diff --git a/kmod/.gitignore b/kmod/.gitignore new file mode 100644 index 00000000..ad8b4ca1 --- /dev/null +++ b/kmod/.gitignore @@ -0,0 +1,16 @@ +*.o +*.ko +*.mod.c +*.cmd +*~ +src/.tmp_versions/ +src/Module.symvers +src/modules.order +cscope.* +*.spec +*.sw[po] +rpmbuild/ +rpms/ + +scoutfs-*.git*/ +scoutfs-kmod-*.tar diff --git a/kmod/Makefile b/kmod/Makefile new file mode 100644 index 00000000..e6d5c979 --- /dev/null +++ b/kmod/Makefile @@ -0,0 +1,57 @@ +ALL: module + +# default to building against the installed source for the running kernel +ifeq ($(SK_KSRC),) +SK_KSRC := $(shell echo /lib/modules/`uname -r`/build) +endif + +# fail if sparse fails if we find it +ifeq ($(shell sparse && echo found),found) +SP = +else +SP = @: +endif + +SCOUTFS_GIT_DESCRIBE := \ + $(shell git describe --all --abbrev=6 --long 2>/dev/null || \ + echo no-git) + +SCOUTFS_FORMAT_HASH := \ + $(shell cat src/format.h src/ioctl.h | md5sum | cut -b1-16) + +SCOUTFS_ARGS := SCOUTFS_GIT_DESCRIBE=$(SCOUTFS_GIT_DESCRIBE) \ + SCOUTFS_FORMAT_HASH=$(SCOUTFS_FORMAT_HASH) \ + CONFIG_SCOUTFS_FS=m -C $(SK_KSRC) M=$(CURDIR)/src \ + EXTRA_CFLAGS="-Werror" + +# - We use the git describe from tags to set up the RPM versioning +RPM_VERSION := $(shell git describe --long --tags | awk -F '-' '{gsub(/^v/,""); print $$1}') +RPM_GITHASH := $(shell git rev-parse --short HEAD) +TARFILE = scoutfs-kmod-$(RPM_VERSION).tar + + +.PHONY: .FORCE + +all: module + +module: + make $(SCOUTFS_ARGS) + $(SP) make C=2 CF="-D__CHECK_ENDIAN__" $(SCOUTFS_ARGS) + + +modules_install: + make $(SCOUTFS_ARGS) modules_install + + +%.spec: %.spec.in .FORCE + sed -e 's/@@VERSION@@/$(RPM_VERSION)/g' \ + -e 's/@@GITHASH@@/$(RPM_GITHASH)/g' < $< > $@+ + mv $@+ $@ + + +dist: scoutfs-kmod.spec + git archive --format=tar --prefix scoutfs-kmod-$(RPM_VERSION)/ HEAD^{tree} > $(TARFILE) + @ tar rf $(TARFILE) --transform="s@\(.*\)@scoutfs-$(RPM_VERSION)/\1@" scoutfs-kmod.spec + +clean: + make $(SCOUTFS_ARGS) clean diff --git a/kmod/README.md b/kmod/README.md new file mode 100644 index 00000000..0fa85e3e --- /dev/null +++ b/kmod/README.md @@ -0,0 +1,133 @@ +# Introduction + +scoutfs is a clustered in-kernel Linux filesystem designed and built +from the ground up to support large archival systems. + +Its key differentiating features are: + + - Integrated consistent indexing accelerates archival maintenance operations + - Log-structured commits allow nodes to write concurrently without contention + +It meets best of breed expectations: + + * Fully consistent POSIX semantics between nodes + * Rich metadata to ensure the integrity of metadata references + * Atomic transactions to maintain consistent persistent structures + * First class kernel implementation for high performance and low latency + * Open GPLv2 implementation + +Learn more in the [white paper](https://docs.wixstatic.com/ugd/aaa89b_88a5cc84be0b4d1a90f60d8900834d28.pdf). + +# Current Status + +**Alpha Open Source Development** + +scoutfs is under heavy active development. We're developing it in the +open to give the community an opportunity to affect the design and +implementation. + +The core architectural design elements are in place. Much surrounding +functionality hasn't been implemented. It's appropriate for early +adopters and interested developers, not for production use. + +In that vein, expect significant incompatible changes to both the format +of network messages and persistent structures. To avoid mistakes the +implementation currently calculates a hash of the format and ioctl +header files in the source tree. The kernel module will refuse to mount +a volume created by userspace utilities with a mismatched hash, and it +will refuse to connect to a remote node with a mismatched hash. This +means having to unmount, mkfs, and remount everything across many +functional changes. Once the format is nailed down we'll wire up +forward and back compat machinery and remove this temporary safety +measure. + +The current kernel module is developed against the RHEL/CentOS 7.x +kernel to minimize the friction of developing and testing with partners' +existing infrastructure. Once we're happy with the design we'll shift +development to the upstream kernel while maintaining distro +compatibility branches. + +# Community Mailing List + +Please join us on the open scoutfs-devel@scoutfs.org [mailing list +hosted on Google Groups](https://groups.google.com/a/scoutfs.org/forum/#!forum/scoutfs-devel) +for all discussion of scoutfs. + +# Quick Start + +**This following a very rough example of the procedure to get up and +running, experience will be needed to fill in the gaps. We're happy to +help on the mailing list.** + +The requirements for running scoutfs on a small cluster are: + + 1. One or more nodes running x86-64 CentOS/RHEL 7.4 (or 7.3) + 2. Access to two shared block devices + 3. IPv4 connectivity between the nodes + +The steps for getting scoutfs mounted and operational are: + + 1. Get the kernel module running on the nodes + 2. Make a new filesystem on the devices with the userspace utilities + 3. Mount the devices on all the nodes + +In this example we run all of these commands on three nodes. The names +of the block devices are the same on all the nodes. + +1. Get the Kernel Module and Userspace Binaries + + * Either use snapshot RPMs built from git by Versity: + + ```shell + rpm -i https://scoutfs.s3-us-west-2.amazonaws.com/scoutfs-repo-0.0.1-1.el7_4.noarch.rpm + yum install scoutfs-utils kmod-scoutfs + ``` + + * Or use the binaries built from checked out git repositories: + + ```shell + yum install kernel-devel + git clone git@github.com:versity/scoutfs-kmod-dev.git + make -C scoutfs-kmod-dev module + modprobe libcrc32c + insmod scoutfs-kmod-dev/src/scoutfs.ko + + git clone git@github.com:versity/scoutfs-utils-dev.git + make -C scoutfs-utils-dev + alias scoutfs=$PWD/scoutfs-utils-dev/src/scoutfs + ``` + +2. Make a New Filesystem (**destroys contents, no questions asked**) + + We specify that two of our three nodes must be present to form a + quorum for the system to function. + + ```shell + scoutfs mkfs -Q 2 /dev/meta_dev /dev/data_dev + ``` + +3. Mount the Filesystem + + Each mounting node provides its local IP address on which it will run + an internal server for the other mounts if it is elected the leader by + the quorum. + + ```shell + mkdir /mnt/scoutfs + mount -t scoutfs -o server_addr=$NODE_ADDR,metadev_path=/dev/meta_dev /dev/data_dev /mnt/scoutfs + ``` + +4. For Kicks, Observe the Metadata Change Index + + The `meta_seq` index tracks the inodes that are changed in each + transaction. + + ```shell + scoutfs walk-inodes meta_seq 0 -1 /mnt/scoutfs + touch /mnt/scoutfs/one; sync + scoutfs walk-inodes meta_seq 0 -1 /mnt/scoutfs + touch /mnt/scoutfs/two; sync + scoutfs walk-inodes meta_seq 0 -1 /mnt/scoutfs + touch /mnt/scoutfs/one; sync + scoutfs walk-inodes meta_seq 0 -1 /mnt/scoutfs + ``` diff --git a/kmod/scoutfs-kmod.spec.in b/kmod/scoutfs-kmod.spec.in new file mode 100644 index 00000000..ee1acf54 --- /dev/null +++ b/kmod/scoutfs-kmod.spec.in @@ -0,0 +1,80 @@ +%define kmod_name scoutfs +%define kmod_version @@VERSION@@ +%define kmod_git_hash @@GITHASH@@ +%define pkg_date %(date +%%Y%%m%%d) + +# take kernel version or default to uname -r +%{!?kversion: %global kversion %(uname -r)} +%global kernel_version %{kversion} + +%global kernel_source() /usr/src/kernels/%{kernel_version}.$(arch) +%global kernel_release() %{kversion} + +%{!?_release: %global _release 0.%{pkg_date}git%{kmod_git_hash}} + +Name: %{kmod_name} +Summary: %{kmod_name} kernel module +Version: %{kmod_version} +Release: %{_release}%{?dist} +License: GPLv2 +Group: System/Kernel +URL: http://scoutfs.org/ + +BuildRequires: %{kernel_module_package_buildreqs} +BuildRequires: git +BuildRequires: kernel-devel-uname-r = %{kernel_version} +BuildRequires: module-init-tools + +ExclusiveArch: x86_64 + +Source: %{kmod_name}-kmod-%{kmod_version}.tar + +# Build only for standard kernel variant(s); for debug packages, append "debug" +# after "default" (separated by space) +%kernel_module_package default + +# Disable the building of the debug package(s). +%define debug_package %{nil} + +%global install_mod_dir extra/%{name} + + +%description +%{kmod_name} - kernel module + + +%prep +%setup -q -n %{kmod_name}-kmod-%{kmod_version} + +set -- * +mkdir source +mv "$@" source/ +mkdir obj + + +%build +echo "Building for kernel: %{kernel_version} flavors: '%{flavors_to_build}'" +for flavor in %flavors_to_build; do + rm -rf obj/$flavor + cp -r source obj/$flavor + make SK_KSRC=%{kernel_source $flavor} -C obj/$flavor module +done + +%install +export INSTALL_MOD_PATH=$RPM_BUILD_ROOT +export INSTALL_MOD_DIR=%{install_mod_dir} +mkdir -p %{install_mod_dir} +for flavor in %{flavors_to_build}; do + export KSRC=%{kernel_source $flavor} + export KVERSION=%{kernel_release $KSRC} + install -d $INSTALL_MOD_PATH/lib/modules/$KVERSION/%{install_mod_dir} + cp $PWD/obj/$flavor/src/scoutfs.ko $INSTALL_MOD_PATH/lib/modules/$KVERSION/%{install_mod_dir}/ +done + +# mark modules executable so that strip-to-file can strip them +find %{buildroot} -type f -name \*.ko -exec %{__chmod} u+x \{\} \; + + +%clean +rm -rf %{buildroot} + diff --git a/kmod/src/Kconfig b/kmod/src/Kconfig new file mode 100644 index 00000000..eb097405 --- /dev/null +++ b/kmod/src/Kconfig @@ -0,0 +1,10 @@ +config SCOUTFS_FS + tristate "scoutfs filesystem" + help + scoutfs is a clustered file system that stores data in large + blocks in shared block storage. + + To compile this file system support as a module, choose M here. The + module will be called scoutfs. + + If unsure, say N. diff --git a/kmod/src/Makefile b/kmod/src/Makefile new file mode 100644 index 00000000..bd50ec38 --- /dev/null +++ b/kmod/src/Makefile @@ -0,0 +1,63 @@ +obj-$(CONFIG_SCOUTFS_FS) := scoutfs.o + +CFLAGS_super.o = -DSCOUTFS_GIT_DESCRIBE=\"$(SCOUTFS_GIT_DESCRIBE)\" \ + -DSCOUTFS_FORMAT_HASH=0x$(SCOUTFS_FORMAT_HASH)LLU + +CFLAGS_scoutfs_trace.o = -I$(src) # define_trace.h double include + +# add EXTRA_CFLAGS defines for kernel compat +-include $(src)/Makefile.kernelcompat + +scoutfs-y += \ + avl.o \ + alloc.o \ + block.o \ + btree.o \ + client.o \ + counters.o \ + data.o \ + dir.o \ + export.o \ + ext.o \ + file.o \ + forest.o \ + inode.o \ + ioctl.o \ + item.o \ + lock.o \ + lock_server.o \ + msg.o \ + net.o \ + options.o \ + per_task.o \ + quorum.o \ + scoutfs_trace.o \ + server.o \ + sort_priv.o \ + spbm.o \ + srch.o \ + super.o \ + sysfs.o \ + trans.o \ + triggers.o \ + tseq.o \ + xattr.o + +# +# The raw types aren't available in userspace headers. Make sure all +# the types we use in the headers are the exported __ versions. +# +# XXX dunno how we're really supposed to do this in kbuild +# +.PHONY: $(src)/check_exported_types +$(src)/check_exported_types: + @if egrep '\<[us](8|16|32|64\>)' $(src)/format.h $(src)/ioctl.h; then \ + echo "no raw types in exported headers, preface with __"; \ + exit 1; \ + fi + @if egrep '\<__packed\>' $(src)/format.h $(src)/ioctl.h; then \ + echo "no __packed allowed in exported headers"; \ + exit 1; \ + fi + +extra-y += check_exported_types diff --git a/kmod/src/Makefile.kernelcompat b/kmod/src/Makefile.kernelcompat new file mode 100644 index 00000000..bd236c43 --- /dev/null +++ b/kmod/src/Makefile.kernelcompat @@ -0,0 +1,36 @@ +# +# We try to detect the specific api incompatibilities with simple tests +# because distros regularly backport features without changing the +# version. +# + +ccflags-y += -include $(src)/kernelcompat.h + +# +# v3.10-rc6-21-gbb6f619b3a49 +# +# _readdir changes from fop->readdir() to fop->iterate() and from +# filldir(dirent) to dir_emit(ctx). +# +ifneq (,$(shell grep 'iterate.*dir_context' include/linux/fs.h)) +ccflags-y += -DKC_ITERATE_DIR_CONTEXT +endif + +# +# v3.10-rc6-23-g5f99f4e79abc +# +# Helpers including dir_emit_dots() are added in the process of +# switching dcache_readdir() from fop->readdir() to fop->iterate() +# +ifneq (,$(shell grep 'dir_emit_dots' include/linux/fs.h)) +ccflags-y += -DKC_DIR_EMIT_DOTS +endif + +# +# RHEL extended the fop struct so to use it we have to set +# a flag to indicate that the struct is large enough and +# contains the pointer. +# +ifneq (,$(shell grep 'FMODE_KABI_ITERATE' include/linux/fs.h)) +ccflags-y += -DKC_FMODE_KABI_ITERATE +endif diff --git a/kmod/src/alloc.c b/kmod/src/alloc.c new file mode 100644 index 00000000..3f0c6aaa --- /dev/null +++ b/kmod/src/alloc.c @@ -0,0 +1,1239 @@ +/* + * Copyright (C) 2020 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include +#include +#include +#include + +#include "super.h" +#include "block.h" +#include "btree.h" +#include "trans.h" +#include "alloc.h" +#include "counters.h" +#include "scoutfs_trace.h" + +/* + * The core allocator uses extent items in btrees rooted in the super. + * Each free extent is stored in two items. The first item is indexed + * by block location and is used to merge adjacent extents when freeing. + * The second item is indexed by length and is used to find large + * extents to allocate from. + * + * Free extent always consumes the front of the largest extent. This + * attempts to discourage fragmentation by given smaller freed extents + * time for an adjacent free to merge before we attempt to re-use them. + * + * The metadata btrees that store extents are updated with cow. This + * requires allocation during extent item modification on behalf of + * allocation. Avoiding this recursion introduces the second structure, + * persistent singly linked lists of individual blknos. + * + * The alloc lists are used for metadata allocation during a + * transaction. Before each transaction lists of blknos are prepared + * for use during the transaction. This ensures a small predictable + * number of cows needed to fully dirty the metadata allocator + * structures during the transaction. As the transaction proceeds + * allocations are made from a list of available meta blknos, and frees + * are performed by adding blknos to another list of freed blknos. + * After transactions these lists are merged back in to extents. + * + * Data allocations are performed directly on a btree of extent items, + * with a bit of caching to stream small file data allocations from + * memory instead of performing multiple btree calls per block + * allocation. + * + * Every transaction has exclusive access to its metadata list blocks + * and data extent trees which are prepared by the server. For client + * metadata and srch transactions the server moved extents and blocks + * into persistent items that are communicated with the server. For + * server transactions metadata the server has to prepare structures for + * itself. To avoid modifying the same structure both explicitly + * (refilling an allocator) and implicitly (using the current allocator + * for cow allocations), it double buffers list blocks. It uses current + * blocks to modify the next blocks, and swaps them at each transaction. + */ + +/* + * Free extents don't have flags and are stored in two indexes sorted by + * block location and by length, largest first. The block location key + * is set to the final block in the extent so that we can find + * intersections by calling _next() iterators starting with the block + * we're searching for. + */ +static void init_ext_key(struct scoutfs_key *key, int type, u64 start, u64 len) +{ + *key = (struct scoutfs_key) { + .sk_zone = SCOUTFS_FREE_EXTENT_ZONE, + .sk_type = type, + }; + + if (type == SCOUTFS_FREE_EXTENT_BLKNO_TYPE) { + key->skfb_end = cpu_to_le64(start + len - 1); + key->skfb_len = cpu_to_le64(len); + } else if (type == SCOUTFS_FREE_EXTENT_LEN_TYPE) { + key->skfl_neglen = cpu_to_le64(-len); + key->skfl_blkno = cpu_to_le64(start); + } else { + BUG(); + } +} + +static void ext_from_key(struct scoutfs_extent *ext, struct scoutfs_key *key) +{ + if (key->sk_type == SCOUTFS_FREE_EXTENT_BLKNO_TYPE) { + ext->start = le64_to_cpu(key->skfb_end) - + le64_to_cpu(key->skfb_len) + 1; + ext->len = le64_to_cpu(key->skfb_len); + } else { + ext->start = le64_to_cpu(key->skfl_blkno); + ext->len = -le64_to_cpu(key->skfl_neglen); + } + ext->map = 0; + ext->flags = 0; +} + +struct alloc_ext_args { + struct scoutfs_alloc *alloc; + struct scoutfs_block_writer *wri; + struct scoutfs_alloc_root *root; + int type; +}; + +static int alloc_ext_next(struct super_block *sb, void *arg, + u64 start, u64 len, struct scoutfs_extent *ext) +{ + struct alloc_ext_args *args = arg; + SCOUTFS_BTREE_ITEM_REF(iref); + struct scoutfs_key key; + int ret; + + init_ext_key(&key, args->type, start, len); + + ret = scoutfs_btree_next(sb, &args->root->root, &key, &iref); + if (ret == 0) { + if (iref.val_len != 0) + ret = -EIO; + else if (iref.key->sk_type != args->type) + ret = -ENOENT; + else + ext_from_key(ext, iref.key); + scoutfs_btree_put_iref(&iref); + } + + if (ret < 0) + memset(ext, 0, sizeof(struct scoutfs_extent)); + + return ret; +} + +static int other_type(int type) +{ + if (type == SCOUTFS_FREE_EXTENT_BLKNO_TYPE) + return SCOUTFS_FREE_EXTENT_LEN_TYPE; + else if (type == SCOUTFS_FREE_EXTENT_LEN_TYPE) + return SCOUTFS_FREE_EXTENT_BLKNO_TYPE; + else + BUG(); +} + +/* + * Insert an extent along with its matching item which is indexed by + * opposite of its len or blkno. If we succeed we update the root's + * record of the total length of all the stored extents. + */ +static int alloc_ext_insert(struct super_block *sb, void *arg, + u64 start, u64 len, u64 map, u8 flags) +{ + struct alloc_ext_args *args = arg; + struct scoutfs_key other; + struct scoutfs_key key; + int ret; + int err; + + /* allocator extents don't have mappings or flags */ + if (WARN_ON_ONCE(map || flags)) + return -EINVAL; + + init_ext_key(&key, args->type, start, len); + init_ext_key(&other, other_type(args->type), start, len); + + ret = scoutfs_btree_insert(sb, args->alloc, args->wri, + &args->root->root, &key, NULL, 0); + if (ret == 0) { + ret = scoutfs_btree_insert(sb, args->alloc, args->wri, + &args->root->root, &other, NULL, 0); + if (ret < 0) { + err = scoutfs_btree_delete(sb, args->alloc, args->wri, + &args->root->root, &key); + BUG_ON(err); + } else { + le64_add_cpu(&args->root->total_len, len); + } + } + + return ret; +} + +static int alloc_ext_remove(struct super_block *sb, void *arg, + u64 start, u64 len, u64 map, u8 flags) +{ + struct alloc_ext_args *args = arg; + struct scoutfs_key other; + struct scoutfs_key key; + int ret; + int err; + + init_ext_key(&key, args->type, start, len); + init_ext_key(&other, other_type(args->type), start, len); + + ret = scoutfs_btree_delete(sb, args->alloc, args->wri, + &args->root->root, &key); + if (ret == 0) { + ret = scoutfs_btree_delete(sb, args->alloc, args->wri, + &args->root->root, &other); + if (ret < 0) { + err = scoutfs_btree_insert(sb, args->alloc, args->wri, + &args->root->root, &key, + NULL, 0); + BUG_ON(err); + } else { + le64_add_cpu(&args->root->total_len, -len); + } + } + + return ret; +} + +static struct scoutfs_ext_ops alloc_ext_ops = { + .next = alloc_ext_next, + .insert = alloc_ext_insert, + .remove = alloc_ext_remove, +}; + +static bool invalid_extent(u64 start, u64 end, u64 first, u64 last) +{ + return start > end || start < first || end > last; +} + +static bool invalid_meta_blkno(struct super_block *sb, u64 blkno) +{ + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + + return invalid_extent(blkno, blkno, + le64_to_cpu(super->first_meta_blkno), + le64_to_cpu(super->last_meta_blkno)); +} + +static bool invalid_data_extent(struct super_block *sb, u64 start, u64 len) +{ + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + + return invalid_extent(start, start + len - 1, + le64_to_cpu(super->first_data_blkno), + le64_to_cpu(super->last_data_blkno)); +} + +void scoutfs_alloc_init(struct scoutfs_alloc *alloc, + struct scoutfs_alloc_list_head *avail, + struct scoutfs_alloc_list_head *freed) +{ + memset(alloc, 0, sizeof(struct scoutfs_alloc)); + + spin_lock_init(&alloc->lock); + mutex_init(&alloc->mutex); + alloc->avail = *avail; + alloc->freed = *freed; +} + +/* + * We're about to commit the transaction that used this allocator, drop + * its block references. + */ +int scoutfs_alloc_prepare_commit(struct super_block *sb, + struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri) +{ + scoutfs_block_put(sb, alloc->dirty_avail_bl); + alloc->dirty_avail_bl = NULL; + scoutfs_block_put(sb, alloc->dirty_freed_bl); + alloc->dirty_freed_bl = NULL; + + return 0; +} + +static u32 list_block_space(__le32 nr) +{ + return SCOUTFS_ALLOC_LIST_MAX_BLOCKS - le32_to_cpu(nr); +} + +static u64 list_block_peek(struct scoutfs_alloc_list_block *lblk, + unsigned int skip) +{ + BUG_ON(skip >= le32_to_cpu(lblk->nr)); + + return le64_to_cpu(lblk->blknos[le32_to_cpu(lblk->start) + skip]); +} + +/* + * Add a blkno to the array. Typically we append of the array. But we + * can also prepend once there's no more room at the end. Consumers of + * the blocks sort before removing them. + */ +static void list_block_add(struct scoutfs_alloc_list_head *lhead, + struct scoutfs_alloc_list_block *lblk, u64 blkno) +{ + u32 start = le32_to_cpu(lblk->start); + u32 nr = le32_to_cpu(lblk->nr); + + BUG_ON(lhead->ref.blkno != lblk->hdr.blkno); + BUG_ON(list_block_space(lblk->nr) == 0); + + if (start + nr < SCOUTFS_ALLOC_LIST_MAX_BLOCKS) { + lblk->blknos[start + nr] = cpu_to_le64(blkno); + } else { + start--; + lblk->blknos[start] = cpu_to_le64(blkno); + lblk->start = cpu_to_le32(start); + } + + le32_add_cpu(&lblk->nr, 1); + le64_add_cpu(&lhead->total_nr, 1); + le32_add_cpu(&lhead->first_nr, 1); +} + +/* + * Remove blknos from the start of the array. + */ +static void list_block_remove(struct scoutfs_alloc_list_head *lhead, + struct scoutfs_alloc_list_block *lblk, + unsigned int count) +{ + BUG_ON(lhead->ref.blkno != lblk->hdr.blkno); + BUG_ON(count > SCOUTFS_ALLOC_LIST_MAX_BLOCKS); + BUG_ON(le32_to_cpu(lblk->nr) < count); + + le32_add_cpu(&lblk->nr, -count); + if (lblk->nr == 0) + lblk->start = 0; + else + le32_add_cpu(&lblk->start, count); + le64_add_cpu(&lhead->total_nr, -(u64)count); + le32_add_cpu(&lhead->first_nr, -count); +} + +static int cmp_le64(const void *A, const void *B) +{ + const __le64 *a = A; + const __le64 *b = B; + + return scoutfs_cmp_u64s(le64_to_cpu(*a), le64_to_cpu(*b)); +} + +static void swap_le64(void *A, void *B, int size) +{ + __le64 *a = A; + __le64 *b = B; + + swap(*a, *b); +} + +static void list_block_sort(struct scoutfs_alloc_list_block *lblk) +{ + sort(&lblk->blknos[le32_to_cpu(lblk->start)], le32_to_cpu(lblk->nr), + sizeof(lblk->blknos[0]), cmp_le64, swap_le64); +} + +/* + * We're always reading blocks that we own, so we shouldn't see stale + * references. But the cached block can be stale and we can need to + * invalidate it. + */ +static int read_list_block(struct super_block *sb, + struct scoutfs_alloc_list_ref *ref, + struct scoutfs_block **bl_ret) +{ + struct scoutfs_block *bl = NULL; + + bl = scoutfs_block_read(sb, le64_to_cpu(ref->blkno)); + if (!IS_ERR_OR_NULL(bl) && + !scoutfs_block_consistent_ref(sb, bl, ref->seq, ref->blkno, + SCOUTFS_BLOCK_MAGIC_ALLOC_LIST)) { + scoutfs_inc_counter(sb, alloc_stale_cached_list_block); + scoutfs_block_invalidate(sb, bl); + scoutfs_block_put(sb, bl); + bl = scoutfs_block_read(sb, le64_to_cpu(ref->blkno)); + } + if (IS_ERR(bl)) { + *bl_ret = NULL; + return PTR_ERR(bl); + } + + *bl_ret = bl; + return 0; +} + +/* + * Give the caller a dirty list block, always allocating a new block if + * the ref is empty. + * + * If the caller gives us an allocated blkno for the cow then we know + * that they're taking care of allocating and freeing the blknos, if not + * we call meta alloc and free. + */ +static int dirty_list_block(struct super_block *sb, + struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_alloc_list_ref *ref, + u64 dirty, u64 *old, + struct scoutfs_block **bl_ret) +{ + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + struct scoutfs_block *cow_bl = NULL; + struct scoutfs_block *bl = NULL; + struct scoutfs_alloc_list_block *lblk; + bool undo_alloc = false; + u64 blkno; + int ret; + int err; + + blkno = le64_to_cpu(ref->blkno); + if (blkno) { + ret = read_list_block(sb, ref, &bl); + if (ret < 0) + goto out; + + if (scoutfs_block_writer_is_dirty(sb, bl)) { + ret = 0; + goto out; + } + } + + if (dirty == 0) { + ret = scoutfs_alloc_meta(sb, alloc, wri, &dirty); + if (ret < 0) + goto out; + undo_alloc = true; + } + + cow_bl = scoutfs_block_create(sb, dirty); + if (IS_ERR(cow_bl)) { + ret = PTR_ERR(cow_bl); + goto out; + } + + if (old) { + *old = blkno; + } else if (blkno) { + ret = scoutfs_free_meta(sb, alloc, wri, blkno); + if (ret < 0) + goto out; + } + + if (bl) + memcpy(cow_bl->data, bl->data, SCOUTFS_BLOCK_LG_SIZE); + else + memset(cow_bl->data, 0, SCOUTFS_BLOCK_LG_SIZE); + scoutfs_block_put(sb, bl); + bl = cow_bl; + cow_bl = NULL; + + lblk = bl->data; + lblk->hdr.magic = cpu_to_le32(SCOUTFS_BLOCK_MAGIC_ALLOC_LIST); + lblk->hdr.fsid = super->hdr.fsid; + lblk->hdr.blkno = cpu_to_le64(bl->blkno); + prandom_bytes(&lblk->hdr.seq, sizeof(lblk->hdr.seq)); + + ref->blkno = lblk->hdr.blkno; + ref->seq = lblk->hdr.seq; + + scoutfs_block_writer_mark_dirty(sb, wri, bl); + ret = 0; + +out: + scoutfs_block_put(sb, cow_bl); + if (ret < 0 && undo_alloc) { + err = scoutfs_free_meta(sb, alloc, wri, dirty); + BUG_ON(err); /* inconsistent */ + } + + if (ret < 0) { + scoutfs_block_put(sb, bl); + bl = NULL; + } + *bl_ret = bl; + + return ret; +} + +/* Allocate a new dirty list block if we fill up more than 3/4 of the block. */ +#define EMPTY_FREED_THRESH (SCOUTFS_ALLOC_LIST_MAX_BLOCKS / 4) + +/* + * Get dirty avail and freed list blocks that will be used for meta + * allocations during our transaction. We peek at the next avail blknos + * for the cow allocations and manually record the cow frees rather than + * recursively calling into alloc_meta and free_meta. + * + * In the client the server will have emptied the freed list so it will + * always allocate a new first empty block for frees. But in the server + * it might have long lists of frees that it's trying to merge in to + * extents over multiple transactions. If the head of the freed list + * doesn't have room we add a new empty block. + */ +static int dirty_alloc_blocks(struct super_block *sb, + struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri) +{ + struct scoutfs_alloc_list_ref orig_freed; + struct scoutfs_alloc_list_block *lblk; + struct scoutfs_block *av_bl = NULL; + struct scoutfs_block *fr_bl = NULL; + struct scoutfs_block *bl; + bool link_orig = false; + u64 av_peek; + u64 av_old; + u64 fr_peek; + u64 fr_old; + int ret; + + if (alloc->dirty_avail_bl != NULL) + return 0; + + mutex_lock(&alloc->mutex); + + /* undo dirty freed if we get an error after */ + orig_freed = alloc->freed.ref; + + if (alloc->dirty_avail_bl != NULL) { + ret = 0; + goto out; + } + + /* caller must ensure that transactions commit before running out */ + if (WARN_ON_ONCE(alloc->avail.ref.blkno == 0) || + WARN_ON_ONCE(le32_to_cpu(alloc->avail.first_nr) < 2)) { + ret = -ENOSPC; + goto out; + } + + ret = read_list_block(sb, &alloc->avail.ref, &bl); + if (ret < 0) + goto out; + + lblk = bl->data; + av_peek = list_block_peek(lblk, 0); + fr_peek = list_block_peek(lblk, 1); + scoutfs_block_put(sb, bl); + lblk = NULL; + + if (alloc->freed.ref.blkno && + list_block_space(alloc->freed.first_nr) < EMPTY_FREED_THRESH) { + /* zero ref to force alloc of new block... */ + memset(&alloc->freed.ref, 0, sizeof(alloc->freed.ref)); + alloc->freed.first_nr = 0; + link_orig = true; + } + + /* dirty the first free block */ + ret = dirty_list_block(sb, alloc, wri, &alloc->freed.ref, + fr_peek, &fr_old, &fr_bl); + if (ret < 0) + goto out; + + if (link_orig) { + /* .. and point the new block at the rest of the list */ + lblk = fr_bl->data; + lblk->next = orig_freed; + lblk = NULL; + } + + ret = dirty_list_block(sb, alloc, wri, &alloc->avail.ref, + av_peek, &av_old, &av_bl); + if (ret < 0) + goto out; + + list_block_remove(&alloc->avail, av_bl->data, 2); + /* sort dirty avail to encourage contiguous sorted meta blocks */ + list_block_sort(av_bl->data); + + if (av_old) + list_block_add(&alloc->freed, fr_bl->data, av_old); + if (fr_old) + list_block_add(&alloc->freed, fr_bl->data, fr_old); + + alloc->dirty_avail_bl = av_bl; + av_bl = NULL; + alloc->dirty_freed_bl = fr_bl; + fr_bl = NULL; + ret = 0; + +out: + if (ret < 0 && alloc->freed.ref.blkno != orig_freed.blkno) { + if (fr_bl) + scoutfs_block_writer_forget(sb, wri, fr_bl); + alloc->freed.ref = orig_freed; + } + + mutex_unlock(&alloc->mutex); + scoutfs_block_put(sb, av_bl); + scoutfs_block_put(sb, fr_bl); + return ret; +} + +/* + * Alloc a metadata block for a transaction in either the client or the + * server. The list block in the allocator was prepared for the transaction. + */ +int scoutfs_alloc_meta(struct super_block *sb, struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, u64 *blkno) +{ + struct scoutfs_alloc_list_block *lblk; + int ret; + + ret = dirty_alloc_blocks(sb, alloc, wri); + if (ret < 0) + goto out; + + spin_lock(&alloc->lock); + lblk = alloc->dirty_avail_bl->data; + if (WARN_ON_ONCE(lblk->nr == 0)) { + /* shouldn't happen, transaction should commit first */ + ret = -ENOSPC; + } else { + *blkno = list_block_peek(lblk, 0); + list_block_remove(&alloc->avail, lblk, 1); + ret = 0; + } + spin_unlock(&alloc->lock); + +out: + if (ret < 0) + *blkno = 0; + scoutfs_inc_counter(sb, alloc_alloc_meta); + trace_scoutfs_alloc_alloc_meta(sb, *blkno, ret); + return ret; +} + +int scoutfs_free_meta(struct super_block *sb, struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, u64 blkno) +{ + struct scoutfs_alloc_list_block *lblk; + int ret; + + if (WARN_ON_ONCE(invalid_meta_blkno(sb, blkno))) + return -EINVAL; + + ret = dirty_alloc_blocks(sb, alloc, wri); + if (ret < 0) + goto out; + + spin_lock(&alloc->lock); + lblk = alloc->dirty_freed_bl->data; + if (WARN_ON_ONCE(list_block_space(lblk->nr) == 0)) { + /* shouldn't happen, transaction should commit first */ + ret = -EIO; + } else { + list_block_add(&alloc->freed, lblk, blkno); + ret = 0; + } + spin_unlock(&alloc->lock); + +out: + scoutfs_inc_counter(sb, alloc_free_meta); + trace_scoutfs_alloc_free_meta(sb, blkno, ret); + return ret; +} + +/* + * Allocate a data extent. An extent that's smaller than the requested + * size can be returned. + * + * The caller can provide a cached extent that can satisfy allocations + * and will be refilled by allocations. The caller is responsible for + * freeing any remaining cached extent back into persistent items before + * committing. + * + * Unlike meta allocations, the caller is expected to serialize + * allocations from the root. + */ +int scoutfs_alloc_data(struct super_block *sb, struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_alloc_root *root, + struct scoutfs_extent *cached, u64 count, + u64 *blkno_ret, u64 *count_ret) +{ + struct alloc_ext_args args = { + .alloc = alloc, + .wri = wri, + .root = root, + .type = SCOUTFS_FREE_EXTENT_LEN_TYPE, + }; + struct scoutfs_extent ext; + u64 len; + int ret; + + /* large allocations come straight from the allocator */ + if (count >= SCOUTFS_ALLOC_DATA_LG_THRESH) { + ret = scoutfs_ext_alloc(sb, &alloc_ext_ops, &args, + 0, 0, count, &ext); + if (ret < 0) + goto out; + + *blkno_ret = ext.start; + *count_ret = ext.len; + ret = 0; + goto out; + } + + /* smaller allocations come from a cached extent */ + if (cached->len == 0) { + ret = scoutfs_ext_alloc(sb, &alloc_ext_ops, &args, 0, 0, + SCOUTFS_ALLOC_DATA_LG_THRESH, cached); + if (ret < 0) + goto out; + } + + len = min(count, cached->len); + + *blkno_ret = cached->start; + *count_ret = len; + + cached->start += len; + cached->len -= len; + ret = 0; +out: + if (ret < 0) { + if (ret == -ENOENT) + ret = -ENOSPC; + *blkno_ret = 0; + *count_ret = 0; + } + + scoutfs_inc_counter(sb, alloc_alloc_data); + trace_scoutfs_alloc_alloc_data(sb, count, *blkno_ret, *count_ret, ret); + return ret; +} + +/* + * Free data extents into the freed tree that will be reclaimed by the + * server and made available for future allocators only if our + * transaction succeeds. We don't want to overwrite existing data if + * our transaction fails. + * + * Unlike meta allocations, the caller is expected to serialize data + * allocations. + */ +int scoutfs_free_data(struct super_block *sb, struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_alloc_root *root, u64 blkno, u64 count) +{ + struct alloc_ext_args args = { + .alloc = alloc, + .wri = wri, + .root = root, + .type = SCOUTFS_FREE_EXTENT_BLKNO_TYPE, + }; + int ret; + + if (WARN_ON_ONCE(invalid_data_extent(sb, blkno, count))) + return -EINVAL; + + ret = scoutfs_ext_insert(sb, &alloc_ext_ops, &args, blkno, count, 0, 0); + scoutfs_inc_counter(sb, alloc_free_data); + trace_scoutfs_alloc_free_data(sb, blkno, count, ret); + return ret; +} + + +/* + * Move extent items adding up to the requested total length from the + * src to the dst tree. The caller is responsible for locking the + * trees, usually because they're also looking at total_len to decide + * how much to move. + * + * -ENOENT is returned if we run out of extents in the source tree + * before moving the total. + * + * This first pass is not optimal because it performs full btree walks + * per extent. We could optimize this with more clever btree item + * manipulation functions which can iterate through src and dst blocks + * and let callbacks indicate how to change items. + */ +int scoutfs_alloc_move(struct super_block *sb, struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_alloc_root *dst, + struct scoutfs_alloc_root *src, u64 total) +{ + struct alloc_ext_args args = { + .alloc = alloc, + .wri = wri, + }; + struct scoutfs_extent ext; + u64 moved = 0; + int ret = 0; + int err; + + while (moved < total) { + args.root = src; + args.type = SCOUTFS_FREE_EXTENT_LEN_TYPE; + ret = scoutfs_ext_alloc(sb, &alloc_ext_ops, &args, + 0, 0, total - moved, &ext); + if (ret < 0) + break; + + args.root = dst; + args.type = SCOUTFS_FREE_EXTENT_BLKNO_TYPE; + ret = scoutfs_ext_insert(sb, &alloc_ext_ops, &args, ext.start, + ext.len, ext.map, ext.flags); + if (ret < 0) { + args.root = src; + args.type = SCOUTFS_FREE_EXTENT_BLKNO_TYPE; + err = scoutfs_ext_insert(sb, &alloc_ext_ops, &args, + ext.start, ext.len, ext.map, + ext.flags); + BUG_ON(err); /* inconsistent */ + break; + } + + moved += ext.len; + scoutfs_inc_counter(sb, alloc_moved_extent); + } + + scoutfs_inc_counter(sb, alloc_move); + trace_scoutfs_alloc_move(sb, total, moved, ret); + + return ret; +} + +/* + * We only trim one block, instead of looping trimming all, because the + * caller is assuming that we do a fixed amount of work when they check + * that their allocator has enough remaining free blocks for us. + */ +static int trim_empty_first_block(struct super_block *sb, + struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_alloc_list_head *lhead) +{ + struct scoutfs_alloc_list_block *one = NULL; + struct scoutfs_alloc_list_block *two = NULL; + struct scoutfs_block *one_bl = NULL; + struct scoutfs_block *two_bl = NULL; + int ret; + + if (WARN_ON_ONCE(lhead->ref.blkno == 0) || + WARN_ON_ONCE(lhead->first_nr != 0)) + return 0; + + ret = read_list_block(sb, &lhead->ref, &one_bl); + if (ret < 0) + goto out; + one = one_bl->data; + + if (one->next.blkno) { + ret = read_list_block(sb, &one->next, &two_bl); + if (ret < 0) + goto out; + two = two_bl->data; + } + + ret = scoutfs_free_meta(sb, alloc, wri, le64_to_cpu(lhead->ref.blkno)); + if (ret < 0) + goto out; + + lhead->ref = one->next; + lhead->first_nr = two ? two->nr : 0; + ret = 0; +out: + scoutfs_block_put(sb, one_bl); + scoutfs_block_put(sb, two_bl); + return ret; +} + +/* + * True if the allocator has enough free blocks to cow (alloc and free) + * a list block and all the btree blocks that store extent items. + * + * At most, an extent operation can dirty down three paths of the tree + * to modify a blkno item and two distant len items. We can grow and + * split the root, and then those three paths could share blocks but each + * modify two leaf blocks. + */ +static bool list_can_cow(struct super_block *sb, struct scoutfs_alloc *alloc, + struct scoutfs_alloc_root *root) +{ + u32 most = 1 + (1 + 1 + (3 * (1 - root->root.height + 1))); + + if (le32_to_cpu(alloc->avail.first_nr) < most) { + scoutfs_inc_counter(sb, alloc_list_avail_lo); + return false; + } + + if (list_block_space(alloc->freed.first_nr) < most) { + scoutfs_inc_counter(sb, alloc_list_freed_hi); + return false; + } + + return true; +} + +static bool lhead_in_alloc(struct scoutfs_alloc *alloc, + struct scoutfs_alloc_list_head *lhead) +{ + return lhead == &alloc->avail || lhead == &alloc->freed; +} + +/* + * Move free blocks from extent items in the root into only the first + * block in the list towards the target if it's fallen below the lo + * threshold. This can return success without necessarily moving as + * much as was requested if its meta allocator runs low, the caller is + * expected to check the counts and act accordingly. + * + * -ENOSPC is returned if the root runs out of extents before the list + * reaches the target. + */ +int scoutfs_alloc_fill_list(struct super_block *sb, + struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_alloc_list_head *lhead, + struct scoutfs_alloc_root *root, + u64 lo, u64 target) +{ + struct alloc_ext_args args = { + .alloc = alloc, + .wri = wri, + .root = root, + .type = SCOUTFS_FREE_EXTENT_LEN_TYPE, + }; + struct scoutfs_alloc_list_block *lblk; + struct scoutfs_block *bl = NULL; + struct scoutfs_extent ext; + int ret = 0; + int i; + + if (WARN_ON_ONCE(target < lo) || + WARN_ON_ONCE(lo > SCOUTFS_ALLOC_LIST_MAX_BLOCKS) || + WARN_ON_ONCE(target > SCOUTFS_ALLOC_LIST_MAX_BLOCKS) || + WARN_ON_ONCE(lhead_in_alloc(alloc, lhead))) + return -EINVAL; + + if (le32_to_cpu(lhead->first_nr) >= lo) + return 0; + + ret = dirty_list_block(sb, alloc, wri, &lhead->ref, 0, NULL, &bl); + if (ret < 0) + goto out; + lblk = bl->data; + + while (le32_to_cpu(lblk->nr) < target && + list_can_cow(sb, alloc, root)) { + + ret = scoutfs_ext_alloc(sb, &alloc_ext_ops, &args, 0, 0, + target - le32_to_cpu(lblk->nr), &ext); + if (ret < 0) { + if (ret == -ENOENT) + ret = -ENOSPC; + break; + } + + for (i = 0; i < ext.len; i++) + list_block_add(lhead, lblk, ext.start + i); + } + +out: + scoutfs_block_put(sb, bl); + return ret; +} + +/* + * Move blknos from all the blocks in the list into extents in the root, + * removing empty blocks as we go. This can return success and leave blocks + * on the list if its metadata alloc runs out of space. + */ +int scoutfs_alloc_empty_list(struct super_block *sb, + struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_alloc_root *root, + struct scoutfs_alloc_list_head *lhead) +{ + struct alloc_ext_args args = { + .alloc = alloc, + .wri = wri, + .root = root, + .type = SCOUTFS_FREE_EXTENT_BLKNO_TYPE, + }; + struct scoutfs_alloc_list_block *lblk = NULL; + struct scoutfs_block *bl = NULL; + struct scoutfs_extent ext; + int ret = 0; + + if (WARN_ON_ONCE(lhead_in_alloc(alloc, lhead))) + return -EINVAL; + + while (lhead->ref.blkno && list_can_cow(sb, alloc, args.root)) { + + if (lhead->first_nr == 0) { + ret = trim_empty_first_block(sb, alloc, wri, lhead); + if (ret < 0) + break; + + scoutfs_block_put(sb, bl); + bl = NULL; + continue; + } + + if (bl == NULL) { + ret = dirty_list_block(sb, alloc, wri, &lhead->ref, + 0, NULL, &bl); + if (ret < 0) + break; + lblk = bl->data; + + /* sort to encourage forming extents */ + list_block_sort(lblk); + } + + /* combine free blknos into extents and insert them */ + ext.start = list_block_peek(lblk, 0); + ext.len = 1; + while ((le32_to_cpu(lblk->nr) > ext.len) && + (list_block_peek(lblk, ext.len) == ext.start + ext.len)) + ext.len++; + + ret = scoutfs_ext_insert(sb, &alloc_ext_ops, &args, + ext.start, ext.len, 0, 0); + if (ret < 0) + break; + + list_block_remove(lhead, lblk, ext.len); + } + + scoutfs_block_put(sb, bl); + + return ret; +} + +/* + * Insert the source list at the head of the destination list, leaving + * the source empty. + * + * This looks bad because the lists are singly-linked and we have to cow + * the entire src lsit to update its tail block next ref to the start of + * the dst list. + * + * In practice, this isn't a problem because the server only calls this + * with small lists that it's going to use soon. + */ +int scoutfs_alloc_splice_list(struct super_block *sb, + struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_alloc_list_head *dst, + struct scoutfs_alloc_list_head *src) +{ + struct scoutfs_alloc_list_block *lblk; + struct scoutfs_alloc_list_ref *ref; + struct scoutfs_block *prev = NULL; + struct scoutfs_block *bl = NULL; + int ret = 0; + + if (WARN_ON_ONCE(lhead_in_alloc(alloc, dst)) || + WARN_ON_ONCE(lhead_in_alloc(alloc, src))) + return -EINVAL; + + if (src->ref.blkno == 0) + return 0; + + ref = &src->ref; + while (ref->blkno) { + ret = dirty_list_block(sb, alloc, wri, ref, 0, NULL, &bl); + if (ret < 0) + goto out; + + lblk = bl->data; + ref = &lblk->next; + + scoutfs_block_put(sb, prev); + prev = bl; + bl = NULL; + } + + *ref = dst->ref; + dst->ref = src->ref; + dst->first_nr = src->first_nr; + le64_add_cpu(&dst->total_nr, le64_to_cpu(src->total_nr)); + + memset(src, 0, sizeof(struct scoutfs_alloc_list_head)); + ret = 0; +out: + scoutfs_block_put(sb, prev); + scoutfs_block_put(sb, bl); + return ret; +} + +/* + * Returns true if meta avail and free don't have room for the given + * number of alloctions or frees. + */ +bool scoutfs_alloc_meta_low(struct super_block *sb, + struct scoutfs_alloc *alloc, u32 nr) +{ + bool lo; + + spin_lock(&alloc->lock); + lo = le32_to_cpu(alloc->avail.first_nr) < nr || + list_block_space(alloc->freed.first_nr) < nr; + spin_unlock(&alloc->lock); + + return lo; +} + +/* + * Call the callers callback for every persistent allocator structure + * we can find. + */ +int scoutfs_alloc_foreach(struct super_block *sb, + scoutfs_alloc_foreach_cb_t cb, void *arg) +{ + struct scoutfs_btree_ref stale_refs[2] = {{0,}}; + struct scoutfs_btree_ref refs[2] = {{0,}}; + struct scoutfs_super_block *super = NULL; + struct scoutfs_srch_compact *sc; + struct scoutfs_log_trees lt; + SCOUTFS_BTREE_ITEM_REF(iref); + struct scoutfs_key key; + int ret; + + super = kmalloc(sizeof(struct scoutfs_super_block), GFP_NOFS); + sc = kmalloc(sizeof(struct scoutfs_srch_compact), GFP_NOFS); + if (!super || !sc) { + ret = -ENOMEM; + goto out; + } + +retry: + ret = scoutfs_read_super(sb, super); + if (ret < 0) + goto out; + + refs[0] = super->logs_root.ref; + refs[1] = super->srch_root.ref; + + /* all the server allocators */ + ret = cb(sb, arg, SCOUTFS_ALLOC_OWNER_SERVER, 0, true, true, + le64_to_cpu(super->meta_alloc[0].total_len)) ?: + cb(sb, arg, SCOUTFS_ALLOC_OWNER_SERVER, 0, true, true, + le64_to_cpu(super->meta_alloc[1].total_len)) ?: + cb(sb, arg, SCOUTFS_ALLOC_OWNER_SERVER, 0, false, true, + le64_to_cpu(super->data_alloc.total_len)) ?: + cb(sb, arg, SCOUTFS_ALLOC_OWNER_SERVER, 1, true, true, + le64_to_cpu(super->server_meta_avail[0].total_nr)) ?: + cb(sb, arg, SCOUTFS_ALLOC_OWNER_SERVER, 1, true, true, + le64_to_cpu(super->server_meta_avail[1].total_nr)) ?: + cb(sb, arg, SCOUTFS_ALLOC_OWNER_SERVER, 1, true, false, + le64_to_cpu(super->server_meta_freed[0].total_nr)) ?: + cb(sb, arg, SCOUTFS_ALLOC_OWNER_SERVER, 1, true, false, + le64_to_cpu(super->server_meta_freed[1].total_nr)); + if (ret < 0) + goto out; + + /* mount fs transaction allocators */ + scoutfs_key_init_log_trees(&key, 0, 0); + for (;;) { + ret = scoutfs_btree_next(sb, &super->logs_root, &key, &iref); + if (ret == -ENOENT) + break; + if (ret < 0) + goto out; + + if (iref.val_len == sizeof(lt)) { + key = *iref.key; + memcpy(<, iref.val, sizeof(lt)); + } else { + ret = -EIO; + } + scoutfs_btree_put_iref(&iref); + if (ret < 0) + goto out; + + ret = cb(sb, arg, SCOUTFS_ALLOC_OWNER_MOUNT, + le64_to_cpu(key.sklt_rid), true, true, + le64_to_cpu(lt.meta_avail.total_nr)) ?: + cb(sb, arg, SCOUTFS_ALLOC_OWNER_MOUNT, + le64_to_cpu(key.sklt_rid), true, false, + le64_to_cpu(lt.meta_freed.total_nr)) ?: + cb(sb, arg, SCOUTFS_ALLOC_OWNER_MOUNT, + le64_to_cpu(key.sklt_rid), false, true, + le64_to_cpu(lt.data_avail.total_len)) ?: + cb(sb, arg, SCOUTFS_ALLOC_OWNER_MOUNT, + le64_to_cpu(key.sklt_rid), false, false, + le64_to_cpu(lt.data_freed.total_len)); + if (ret < 0) + goto out; + + scoutfs_key_inc(&key); + } + + /* srch compaction allocators */ + memset(&key, 0, sizeof(key)); + key.sk_zone = SCOUTFS_SRCH_ZONE; + key.sk_type = SCOUTFS_SRCH_PENDING_TYPE; + + for (;;) { + /* _PENDING_ and _BUSY_ are last, _next won't see other types */ + ret = scoutfs_btree_next(sb, &super->srch_root, &key, &iref); + if (ret == -ENOENT) + break; + if (ret == 0) { + if (iref.val_len == sizeof(*sc)) { + key = *iref.key; + memcpy(sc, iref.val, iref.val_len); + } else { + ret = -EIO; + } + scoutfs_btree_put_iref(&iref); + } + if (ret < 0) + goto out; + + ret = cb(sb, arg, SCOUTFS_ALLOC_OWNER_SRCH, + le64_to_cpu(sc->id), true, true, + le64_to_cpu(sc->meta_avail.total_nr)) ?: + cb(sb, arg, SCOUTFS_ALLOC_OWNER_SRCH, + le64_to_cpu(sc->id), true, false, + le64_to_cpu(sc->meta_freed.total_nr)); + if (ret < 0) + goto out; + + scoutfs_key_inc(&key); + } + + ret = 0; +out: + if (ret == -ESTALE) { + if (memcmp(&stale_refs, &refs, sizeof(refs)) == 0) { + ret = -EIO; + } else { + BUILD_BUG_ON(sizeof(stale_refs) != sizeof(refs)); + memcpy(stale_refs, refs, sizeof(stale_refs)); + goto retry; + } + } + + kfree(super); + kfree(sc); + return ret; +} diff --git a/kmod/src/alloc.h b/kmod/src/alloc.h new file mode 100644 index 00000000..da8686f7 --- /dev/null +++ b/kmod/src/alloc.h @@ -0,0 +1,135 @@ +#ifndef _SCOUTFS_ALLOC_H_ +#define _SCOUTFS_ALLOC_H_ + +#include "ext.h" + +/* + * These are implementation-specific metrics, they don't need to be + * consistent across implementations. They should probably be run-time + * knobs. + */ + +/* + * The largest extent that we'll try to allocate with fallocate. We're + * trying not to completely consume a transactions data allocation all + * at once. This is only allocation granularity, repeated allocations + * can produce large contiguous extents. + */ +#define SCOUTFS_FALLOCATE_ALLOC_LIMIT \ + (128ULL * 1024 * 1024 >> SCOUTFS_BLOCK_SM_SHIFT) + +/* + * The largest aligned region that we'll try to allocate at the end of + * the file as it's extended. This is also limited to the current file + * size so we can only waste at most twice the total file size when + * files are less than this. We try to keep this around the point of + * diminishing returns in streaming performance of common data devices + * to limit waste. + */ +#define SCOUTFS_DATA_EXTEND_PREALLOC_LIMIT \ + (8ULL * 1024 * 1024 >> SCOUTFS_BLOCK_SM_SHIFT) + +/* + * Small data allocations are satisfied by cached extents stored in + * the run-time alloc struct to minimize item operations for small + * block allocations. Large allocations come directly from btree + * extent items, and this defines the threshold beetwen them. + */ +#define SCOUTFS_ALLOC_DATA_LG_THRESH \ + (8ULL * 1024 * 1024 >> SCOUTFS_BLOCK_SM_SHIFT) + +/* + * Fill client alloc roots to the target when they fall below the lo + * threshold. + * + * We're giving the client the most available meta blocks we can so that + * it has the freedom to build large transactions before worrying that + * it might run out of meta allocs during commits. + */ +#define SCOUTFS_SERVER_META_FILL_TARGET \ + SCOUTFS_ALLOC_LIST_MAX_BLOCKS +#define SCOUTFS_SERVER_META_FILL_LO \ + (SCOUTFS_ALLOC_LIST_MAX_BLOCKS / 2) +#define SCOUTFS_SERVER_DATA_FILL_TARGET \ + (4ULL * 1024 * 1024 * 1024 >> SCOUTFS_BLOCK_SM_SHIFT) +#define SCOUTFS_SERVER_DATA_FILL_LO \ + (1ULL * 1024 * 1024 * 1024 >> SCOUTFS_BLOCK_SM_SHIFT) + +/* + * Each of the server meta_alloc roots will try to keep a minimum amount + * of free blocks. The server will swap roots when its current avail + * falls below the threshold while the freed root is still above it. It + * must have room for all the largest allocation attempted in a + * transaction on the server. + */ +#define SCOUTFS_SERVER_META_ALLOC_MIN \ + (SCOUTFS_SERVER_META_FILL_TARGET * 2) + +/* + * A run-time use of a pair of persistent avail/freed roots as a + * metadata allocator. It has the machinery needed to lock and avoid + * recursion when dirtying the list blocks that are used during the + * transaction. + */ +struct scoutfs_alloc { + spinlock_t lock; + struct mutex mutex; + struct scoutfs_block *dirty_avail_bl; + struct scoutfs_block *dirty_freed_bl; + struct scoutfs_alloc_list_head avail; + struct scoutfs_alloc_list_head freed; +}; + +void scoutfs_alloc_init(struct scoutfs_alloc *alloc, + struct scoutfs_alloc_list_head *avail, + struct scoutfs_alloc_list_head *freed); +int scoutfs_alloc_prepare_commit(struct super_block *sb, + struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri); + +int scoutfs_alloc_meta(struct super_block *sb, struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, u64 *blkno); +int scoutfs_free_meta(struct super_block *sb, struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, u64 blkno); + +int scoutfs_alloc_data(struct super_block *sb, struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_alloc_root *root, + struct scoutfs_extent *cached, u64 count, + u64 *blkno_ret, u64 *count_ret); +int scoutfs_free_data(struct super_block *sb, struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_alloc_root *root, u64 blkno, u64 count); + +int scoutfs_alloc_move(struct super_block *sb, struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_alloc_root *dst, + struct scoutfs_alloc_root *src, u64 total); + +int scoutfs_alloc_fill_list(struct super_block *sb, + struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_alloc_list_head *lhead, + struct scoutfs_alloc_root *root, + u64 lo, u64 target); +int scoutfs_alloc_empty_list(struct super_block *sb, + struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_alloc_root *root, + struct scoutfs_alloc_list_head *lhead); +int scoutfs_alloc_splice_list(struct super_block *sb, + struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_alloc_list_head *dst, + struct scoutfs_alloc_list_head *src); + +bool scoutfs_alloc_meta_low(struct super_block *sb, + struct scoutfs_alloc *alloc, u32 nr); + +typedef int (*scoutfs_alloc_foreach_cb_t)(struct super_block *sb, void *arg, + int owner, u64 id, + bool meta, bool avail, u64 blocks); +int scoutfs_alloc_foreach(struct super_block *sb, + scoutfs_alloc_foreach_cb_t cb, void *arg); + +#endif diff --git a/kmod/src/avl.c b/kmod/src/avl.c new file mode 100644 index 00000000..98c4a0a5 --- /dev/null +++ b/kmod/src/avl.c @@ -0,0 +1,405 @@ +/* + * Copyright (C) 2020 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include + +#include "format.h" +#include "avl.h" + +/* + * We use a simple avl to index items in btree blocks. The interface + * looks a bit like the kernel rbtree interface in that the caller + * manages locking and storage for the nodes. Node references are + * stored as byte offsets from the root so that the implementation + * doesn't have to know anything about the caller's container. + * + * We store the full height in each node, rather than just 2 bits for + * the balance, so that we can use the extra redundancy to verify the + * integrity of the tree. + */ + +static struct scoutfs_avl_node *node_ptr(struct scoutfs_avl_root *root, + __le16 off) +{ + return off ? (void *)root + le16_to_cpu(off) : NULL; +} + +static __le16 node_off(struct scoutfs_avl_root *root, + struct scoutfs_avl_node *node) +{ + return node ? cpu_to_le16((void *)node - (void *)root) : 0; +} + +static __u8 node_height(struct scoutfs_avl_node *node) +{ + return node ? node->height : 0; +} + +struct scoutfs_avl_node * +scoutfs_avl_search(struct scoutfs_avl_root *root, + scoutfs_avl_compare_t compare, void *arg, int *cmp_ret, + struct scoutfs_avl_node **par, + struct scoutfs_avl_node **next, + struct scoutfs_avl_node **prev) +{ + struct scoutfs_avl_node *node = node_ptr(root, root->node); + int cmp; + + if (cmp_ret) + *cmp_ret = -1; + if (par) + *par = NULL; + if (next) + *next = NULL; + if (prev) + *prev = NULL; + + while (node) { + cmp = compare(arg, node); + if (par) + *par = node; + if (cmp_ret) + *cmp_ret = cmp; + if (cmp < 0) { + if (next) + *next = node; + node = node_ptr(root, node->left); + } else if (cmp > 0) { + if (prev) + *prev = node; + node = node_ptr(root, node->right); + } else { + return node; + } + } + + return NULL; +} + +struct scoutfs_avl_node *scoutfs_avl_first(struct scoutfs_avl_root *root) +{ + struct scoutfs_avl_node *node = node_ptr(root, root->node); + + while (node && node->left) + node = node_ptr(root, node->left); + + return node; +} + +struct scoutfs_avl_node *scoutfs_avl_last(struct scoutfs_avl_root *root) +{ + struct scoutfs_avl_node *node = node_ptr(root, root->node); + + while (node && node->right) + node = node_ptr(root, node->right); + + return node; +} + +struct scoutfs_avl_node *scoutfs_avl_next(struct scoutfs_avl_root *root, + struct scoutfs_avl_node *node) +{ + struct scoutfs_avl_node *parent; + + if (node->right) { + node = node_ptr(root, node->right); + while (node->left) + node = node_ptr(root, node->left); + return node; + } + + while ((parent = node_ptr(root, node->parent)) && + node == node_ptr(root, parent->right)) + node = parent; + + return parent; +} + +struct scoutfs_avl_node *scoutfs_avl_prev(struct scoutfs_avl_root *root, + struct scoutfs_avl_node *node) +{ + struct scoutfs_avl_node *parent; + + if (node->left) { + node = node_ptr(root, node->left); + while (node->right) + node = node_ptr(root, node->right); + return node; + } + + while ((parent = node_ptr(root, node->parent)) && + node == node_ptr(root, parent->left)) + node = parent; + + return parent; +} + +static void set_parent_left_right(struct scoutfs_avl_root *root, + struct scoutfs_avl_node *parent, + struct scoutfs_avl_node *old, + struct scoutfs_avl_node *new) +{ + __le16 *off; + + if (parent == NULL) + off = &root->node; + else if (parent->left == node_off(root, old)) + off = &parent->left; + else + off = &parent->right; + + *off = node_off(root, new); +} + +static void set_height(struct scoutfs_avl_root *root, + struct scoutfs_avl_node *node) +{ + struct scoutfs_avl_node *left = node_ptr(root, node->left); + struct scoutfs_avl_node *right = node_ptr(root, node->right); + + node->height = 1 + max(node_height(left), node_height(right)); +} + +static int node_balance(struct scoutfs_avl_root *root, + struct scoutfs_avl_node *node) +{ + if (node == NULL) + return 0; + + return (int)node_height(node_ptr(root, node->right)) - + (int)node_height(node_ptr(root, node->left)); +} + +/* + * d b + * / \ rotate right -> / \ + * b e a d + * / \ <- rotate left / \ + * a c c e + * + * The rotate functions are always called with the higher node as the + * earlier argument. Links to a and e are constant. We have to update + * the forward and back refs between parents and nodes for the three links + * along root->[db]->[bd]->c. + */ +static void rotate_right(struct scoutfs_avl_root *root, + struct scoutfs_avl_node *d) +{ + struct scoutfs_avl_node *gpa = node_ptr(root, d->parent); + struct scoutfs_avl_node *b = node_ptr(root, d->left); + struct scoutfs_avl_node *c = node_ptr(root, b->right); + + set_parent_left_right(root, gpa, d, b); + b->parent = node_off(root, gpa); + + b->right = node_off(root, d); + d->parent = node_off(root, b); + + d->left = node_off(root, c); + if (c) + c->parent = node_off(root, d); + + set_height(root, d); + set_height(root, b); +} + +static void rotate_left(struct scoutfs_avl_root *root, + struct scoutfs_avl_node *b) +{ + struct scoutfs_avl_node *gpa = node_ptr(root, b->parent); + struct scoutfs_avl_node *d = node_ptr(root, b->right); + struct scoutfs_avl_node *c = node_ptr(root, d->left); + + set_parent_left_right(root, gpa, b, d); + d->parent = node_off(root, gpa); + + d->left = node_off(root, b); + b->parent = node_off(root, d); + + b->right = node_off(root, c); + if (c) + c->parent = node_off(root, b); + + set_height(root, b); + set_height(root, d); +} + +/* + * Check the balance factor for the given node and perform rotations if + * its two child subtrees are too far out of balance. Return either the + * node again or the root of the newly balanced subtree. + */ +static struct scoutfs_avl_node * +rotate_imbalance(struct scoutfs_avl_root *root, struct scoutfs_avl_node *node) +{ + int bal = node_balance(root, node); + struct scoutfs_avl_node *child; + + if (bal >= -1 && bal <= 1) + return node; + + if (bal > 0) { + /* turn right-left case into right-right */ + child = node_ptr(root, node->right); + if (node_balance(root, child) < 0) + rotate_right(root, child); + /* rotate left to address right-right */ + rotate_left(root, node); + + } else { + /* or do the mirror for the left- cases */ + child = node_ptr(root, node->left); + if (node_balance(root, child) > 0) + rotate_left(root, child); + rotate_right(root, node); + } + + return node_ptr(root, node->parent); +} + +void scoutfs_avl_insert(struct scoutfs_avl_root *root, + struct scoutfs_avl_node *parent, + struct scoutfs_avl_node *node, int cmp) +{ + node->parent = 0; + node->left = 0; + node->right = 0; + set_height(root, node); + memset(node->__pad, 0, sizeof(node->__pad)); + + if (parent == NULL) { + root->node = node_off(root, node); + node->parent = 0; + return; + } + + if (cmp < 0) + parent->left = node_off(root, node); + else + parent->right = node_off(root, node); + node->parent = node_off(root, parent); + + while (parent) { + set_height(root, parent); + parent = rotate_imbalance(root, parent); + parent = node_ptr(root, parent->parent); + } +} + +static struct scoutfs_avl_node *avl_successor(struct scoutfs_avl_root *root, + struct scoutfs_avl_node *node) +{ + node = node_ptr(root, node->right); + while (node->left) + node = node_ptr(root, node->left); + + return node; +} + +/* + * Find a node next successor and then swap the positions of the two + * nodes with each other in the tree. This is only tricky because the + * successor can be a direct child of the node and if we weren't careful + * we'd be modifying each of the nodes through the pointers between + * them. + */ +static void swap_with_successor(struct scoutfs_avl_root *root, + struct scoutfs_avl_node *node) +{ + struct scoutfs_avl_node *succ = avl_successor(root, node); + struct scoutfs_avl_node *succ_par = node_ptr(root, succ->parent); + struct scoutfs_avl_node *succ_right = node_ptr(root, succ->right); + struct scoutfs_avl_node *parent; + struct scoutfs_avl_node *left; + struct scoutfs_avl_node *right; + + /* Link old node's parent and left child with the successor */ + succ->parent = node->parent; + parent = node_ptr(root, succ->parent); + set_parent_left_right(root, parent, node, succ); + succ->left = node->left; + left = node_ptr(root, succ->left); + if (left) + left->parent = node_off(root, succ); + + /* + * Link the old node's right with successor and the old + * successor's parent with the node, they could have pointed to + * each other. + */ + if (succ_par == node) { + succ->right = node_off(root, node); + node->parent = node_off(root, succ); + } else { + succ->right = node->right; + right = node_ptr(root, succ->right); + if (right) + right->parent = node_off(root, succ); + set_parent_left_right(root, succ_par, succ, node); + node->parent = node_off(root, succ_par); + } + + /* Link the old successor's right with the node, it can't have left */ + node->right = node_off(root, succ_right); + if (succ_right) + succ_right->parent = node_off(root, node); + node->left = 0; + + swap(node->height, succ->height); +} + +void scoutfs_avl_delete(struct scoutfs_avl_root *root, + struct scoutfs_avl_node *node) +{ + struct scoutfs_avl_node *parent; + struct scoutfs_avl_node *child; + + if (node->left && node->right) + swap_with_successor(root, node); + + parent = node_ptr(root, node->parent); + child = node_ptr(root, node->left ?: node->right); + + set_parent_left_right(root, parent, node, child); + if (child) + child->parent = node->parent; + + while (parent) { + set_height(root, parent); + parent = rotate_imbalance(root, parent); + parent = node_ptr(root, parent->parent); + } +} + +/* + * Move the contents of a node to a new node location in memory. The + * logical position of the node in the tree does not change. + */ +void scoutfs_avl_relocate(struct scoutfs_avl_root *root, + struct scoutfs_avl_node *to, + struct scoutfs_avl_node *from) +{ + struct scoutfs_avl_node *parent = node_ptr(root, from->parent); + struct scoutfs_avl_node *left = node_ptr(root, from->left); + struct scoutfs_avl_node *right = node_ptr(root, from->right); + + set_parent_left_right(root, parent, from, to); + to->parent = from->parent; + to->left = from->left; + if (left) + left->parent = node_off(root, to); + to->right = from->right; + if (right) + right->parent = node_off(root, to); + to->height = from->height; +} diff --git a/kmod/src/avl.h b/kmod/src/avl.h new file mode 100644 index 00000000..f50ca423 --- /dev/null +++ b/kmod/src/avl.h @@ -0,0 +1,30 @@ +#ifndef _SCOUTFS_AVL_H_ +#define _SCOUTFS_AVL_H_ + +#include "format.h" + +typedef int (*scoutfs_avl_compare_t)(void *arg, + struct scoutfs_avl_node *node); + +struct scoutfs_avl_node * +scoutfs_avl_search(struct scoutfs_avl_root *root, + scoutfs_avl_compare_t compare, void *arg, int *cmp_ret, + struct scoutfs_avl_node **par, + struct scoutfs_avl_node **next, + struct scoutfs_avl_node **prev); +struct scoutfs_avl_node *scoutfs_avl_first(struct scoutfs_avl_root *root); +struct scoutfs_avl_node *scoutfs_avl_last(struct scoutfs_avl_root *root); +struct scoutfs_avl_node *scoutfs_avl_next(struct scoutfs_avl_root *root, + struct scoutfs_avl_node *node); +struct scoutfs_avl_node *scoutfs_avl_prev(struct scoutfs_avl_root *root, + struct scoutfs_avl_node *node); +void scoutfs_avl_insert(struct scoutfs_avl_root *root, + struct scoutfs_avl_node *parent, + struct scoutfs_avl_node *node, int cmp); +void scoutfs_avl_delete(struct scoutfs_avl_root *root, + struct scoutfs_avl_node *node); +void scoutfs_avl_relocate(struct scoutfs_avl_root *root, + struct scoutfs_avl_node *to, + struct scoutfs_avl_node *from); + +#endif diff --git a/kmod/src/block.c b/kmod/src/block.c new file mode 100644 index 00000000..146925cb --- /dev/null +++ b/kmod/src/block.c @@ -0,0 +1,1001 @@ +/* + * Copyright (C) 2019 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "format.h" +#include "super.h" +#include "block.h" +#include "counters.h" +#include "msg.h" +#include "scoutfs_trace.h" + +/* + * The scoutfs block cache manages metadata blocks that can be larger + * than the page size. Callers can have their own contexts for tracking + * dirty blocks that are written together. We pin dirty blocks in + * memory and only checksum them all as they're all written. + * + * An LRU is maintained so the VM can reclaim the oldest presumably + * unlikely to be used blocks. But we don't maintain a perfect record + * of access order. We only move accessed blocks to the tail of the rcu + * if they weren't in the most recently moved fraction of the total + * population. This means that reclaim will walk through waves of that + * fraction of the population. It's close enough and removes lru + * maintenance locking from the fast path. + */ + +struct block_info { + struct super_block *sb; + spinlock_t lock; + struct radix_tree_root radix; + struct list_head lru_list; + u64 lru_nr; + u64 lru_move_counter; + wait_queue_head_t waitq; + struct shrinker shrinker; + struct work_struct free_work; + struct llist_head free_llist; +}; + +#define DECLARE_BLOCK_INFO(sb, name) \ + struct block_info *name = SCOUTFS_SB(sb)->block_info + +enum block_status_bits { + BLOCK_BIT_UPTODATE = 0, /* contents consistent with media */ + BLOCK_BIT_NEW, /* newly allocated, contents undefined */ + BLOCK_BIT_DIRTY, /* dirty, writer will write */ + BLOCK_BIT_IO_BUSY, /* bios are in flight */ + BLOCK_BIT_ERROR, /* saw IO error */ + BLOCK_BIT_DELETED, /* has been deleted from radix tree */ + BLOCK_BIT_PAGE_ALLOC, /* page (possibly high order) allocation */ + BLOCK_BIT_VIRT, /* mapped virt allocation */ + BLOCK_BIT_CRC_VALID, /* crc has been verified */ +}; + +struct block_private { + struct scoutfs_block bl; + struct super_block *sb; + atomic_t refcount; + union { + struct list_head lru_entry; + struct llist_node free_node; + }; + u64 lru_moved; + struct list_head dirty_entry; + unsigned long bits; + atomic_t io_count; + union { + struct page *page; + void *virt; + }; +}; + +#define TRACE_BLOCK(which, bp) \ +do { \ + __typeof__(bp) _bp = (bp); \ + trace_scoutfs_block_##which(_bp->sb, _bp, _bp->bl.blkno, \ + atomic_read(&_bp->refcount), \ + atomic_read(&_bp->io_count), \ + _bp->bits, _bp->lru_moved); \ +} while (0) + +#define BLOCK_PRIVATE(_bl) \ + container_of((_bl), struct block_private, bl) + +/* + * These _block_header helpers are from a previous generation and may + * be refactored away. + */ + +__le32 scoutfs_block_calc_crc(struct scoutfs_block_header *hdr, u32 size) +{ + int off = offsetof(struct scoutfs_block_header, crc) + + FIELD_SIZEOF(struct scoutfs_block_header, crc); + u32 calc = crc32c(~0, (char *)hdr + off, size - off); + + return cpu_to_le32(calc); +} + +bool scoutfs_block_valid_crc(struct scoutfs_block_header *hdr, u32 size) +{ + return hdr->crc == scoutfs_block_calc_crc(hdr, size); +} + +bool scoutfs_block_valid_ref(struct super_block *sb, + struct scoutfs_block_header *hdr, + __le64 seq, __le64 blkno) +{ + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + + return hdr->fsid == super->hdr.fsid && hdr->seq == seq && + hdr->blkno == blkno; +} + +static struct block_private *block_alloc(struct super_block *sb, u64 blkno) +{ + struct block_private *bp; + + /* + * If we had multiple blocks per page we'd need to be a little + * more careful with a partial page allocator when allocating + * blocks and would make the lru per-page instead of per-block. + */ + BUILD_BUG_ON(PAGE_SIZE > SCOUTFS_BLOCK_LG_SIZE); + + bp = kzalloc(sizeof(struct block_private), GFP_NOFS); + if (!bp) + goto out; + + bp->page = alloc_pages(GFP_NOFS | __GFP_NOWARN, + SCOUTFS_BLOCK_LG_PAGE_ORDER); + if (bp->page) { + scoutfs_inc_counter(sb, block_cache_alloc_page_order); + set_bit(BLOCK_BIT_PAGE_ALLOC, &bp->bits); + bp->bl.data = page_address(bp->page); + } else { + bp->virt = __vmalloc(SCOUTFS_BLOCK_LG_SIZE, + GFP_NOFS | __GFP_HIGHMEM, PAGE_KERNEL); + if (!bp->virt) { + kfree(bp); + bp = NULL; + goto out; + } + + scoutfs_inc_counter(sb, block_cache_alloc_virt); + set_bit(BLOCK_BIT_VIRT, &bp->bits); + bp->bl.data = bp->virt; + } + + bp->bl.blkno = blkno; + bp->sb = sb; + atomic_set(&bp->refcount, 1); + INIT_LIST_HEAD(&bp->lru_entry); + INIT_LIST_HEAD(&bp->dirty_entry); + set_bit(BLOCK_BIT_NEW, &bp->bits); + atomic_set(&bp->io_count, 0); + + TRACE_BLOCK(allocate, bp); + +out: + if (!bp) + scoutfs_inc_counter(sb, block_cache_alloc_failure); + return bp; +} + +static void block_free(struct super_block *sb, struct block_private *bp) +{ + scoutfs_inc_counter(sb, block_cache_free); + + TRACE_BLOCK(free, bp); + + if (test_bit(BLOCK_BIT_PAGE_ALLOC, &bp->bits)) + __free_pages(bp->page, SCOUTFS_BLOCK_LG_PAGE_ORDER); + else if (test_bit(BLOCK_BIT_VIRT, &bp->bits)) + vfree(bp->virt); + else + BUG(); + + /* lru_entry could have been clobbered by union member free_node */ + WARN_ON_ONCE(!list_empty(&bp->dirty_entry)); + WARN_ON_ONCE(atomic_read(&bp->refcount)); + WARN_ON_ONCE(atomic_read(&bp->io_count)); + kfree(bp); +} + +/* + * We free blocks in task context so we can free kernel virtual mappings. + */ +static void block_free_work(struct work_struct *work) +{ + struct block_info *binf = container_of(work, struct block_info, + free_work); + struct super_block *sb = binf->sb; + struct block_private *bp; + struct llist_node *deleted; + + deleted = llist_del_all(&binf->free_llist); + + llist_for_each_entry(bp, deleted, free_node) { + block_free(sb, bp); + } +} + +/* + * After we've dropped the final ref kick off the final free in task + * context. This happens in the relatively rare cases of IO errors, + * stale cached data, memory pressure, and unmount. + */ +static void block_put(struct super_block *sb, struct block_private *bp) +{ + DECLARE_BLOCK_INFO(sb, binf); + + if (!IS_ERR_OR_NULL(bp) && atomic_dec_and_test(&bp->refcount)) { + WARN_ON_ONCE(!list_empty(&bp->lru_entry)); + llist_add(&bp->free_node, &binf->free_llist); + schedule_work(&binf->free_work); + } +} + +/* + * Add a new block into the cache. The caller holds the lock and has + * preloaded the radix. + */ +static void block_insert(struct super_block *sb, struct block_private *bp, + u64 blkno) +{ + DECLARE_BLOCK_INFO(sb, binf); + + assert_spin_locked(&binf->lock); + BUG_ON(!list_empty(&bp->lru_entry)); + + atomic_inc(&bp->refcount); + radix_tree_insert(&binf->radix, blkno, bp); + list_add_tail(&bp->lru_entry, &binf->lru_list); + bp->lru_moved = ++binf->lru_move_counter; + binf->lru_nr++; + + TRACE_BLOCK(insert, bp); +} + +/* + * Only move the block to the tail of the LRU if it's outside of the + * small fraction of the lru population that has been most recently + * used. This gives us a reasonable number of most recently accessed + * blocks which will be reclaimed after the rest of the least recently + * used blocks while reducing per-access locking overhead of maintaining + * the LRU. We don't care about unlikely non-atomic u64 accesses racing + * and messing up LRU position. + * + * This can race with blocks being removed from the cache (shrinking, + * stale, errors) so we're careful to only move the entry if it's still + * on the list after we acquire the lock. We still hold a reference so it's + * lru_entry hasn't transitioned to being used as the free_node. + */ +static void block_accessed(struct super_block *sb, struct block_private *bp) +{ + DECLARE_BLOCK_INFO(sb, binf); + u64 recent = binf->lru_nr >> 3; + + scoutfs_inc_counter(sb, block_cache_access); + + if (bp->lru_moved < (binf->lru_move_counter - recent)) { + spin_lock(&binf->lock); + if (!list_empty(&bp->lru_entry)) { + list_move_tail(&bp->lru_entry, &binf->lru_list); + bp->lru_moved = ++binf->lru_move_counter; + scoutfs_inc_counter(sb, block_cache_lru_move); + } + spin_unlock(&binf->lock); + } +} + +/* + * Remove a block from the cache and drop its reference. We only remove + * the block once as the deleted bit is first set. + */ +static void block_remove(struct super_block *sb, struct block_private *bp) +{ + DECLARE_BLOCK_INFO(sb, binf); + + assert_spin_locked(&binf->lock); + + if (!test_and_set_bit(BLOCK_BIT_DELETED, &bp->bits)) { + BUG_ON(list_empty(&bp->lru_entry)); + radix_tree_delete(&binf->radix, bp->bl.blkno); + list_del_init(&bp->lru_entry); + binf->lru_nr--; + block_put(sb, bp); + } +} + +static bool io_busy(struct block_private *bp) +{ + smp_rmb(); /* test after adding to wait queue */ + return test_bit(BLOCK_BIT_IO_BUSY, &bp->bits); +} + +/* + * Called during shutdown with no other users. + */ +static void block_remove_all(struct super_block *sb) +{ + DECLARE_BLOCK_INFO(sb, binf); + struct block_private *bp; + + spin_lock(&binf->lock); + + while (radix_tree_gang_lookup(&binf->radix, (void **)&bp, 0, 1) == 1) { + wait_event(binf->waitq, !io_busy(bp)); + block_remove(sb, bp); + } + + spin_unlock(&binf->lock); + + WARN_ON_ONCE(!list_empty(&binf->lru_list)); + WARN_ON_ONCE(binf->lru_nr != 0); + WARN_ON_ONCE(binf->radix.rnode != NULL); +} + +/* + * XXX The io_count and sb fields in the block_private are only used + * during IO. We don't need to have them sitting around for the entire + * lifetime of each cached block. + * + * This is happening in interrupt context so we do as little work as + * possible. Final freeing, verifying checksums, and unlinking errored + * blocks are all done by future users of the blocks. + */ +static void block_end_io(struct super_block *sb, int rw, + struct block_private *bp, int err) +{ + DECLARE_BLOCK_INFO(sb, binf); + bool is_read = !(rw & WRITE); + + if (err) { + scoutfs_inc_counter(sb, block_cache_end_io_error); + set_bit(BLOCK_BIT_ERROR, &bp->bits); + } + + if (!atomic_dec_and_test(&bp->io_count)) + return; + + if (is_read && !test_bit(BLOCK_BIT_ERROR, &bp->bits)) + set_bit(BLOCK_BIT_UPTODATE, &bp->bits); + + clear_bit(BLOCK_BIT_IO_BUSY, &bp->bits); + block_put(sb, bp); + + /* make sure set and cleared bits are visible to woken */ + smp_mb(); + + if (waitqueue_active(&binf->waitq)) + wake_up(&binf->waitq); +} + +static void block_bio_end_io(struct bio *bio, int err) +{ + struct block_private *bp = bio->bi_private; + struct super_block *sb = bp->sb; + + TRACE_BLOCK(end_io, bp); + block_end_io(sb, bio->bi_rw, bp, err); + bio_put(bio); +} + +/* + * Kick off IO for a single block. + */ +static int block_submit_bio(struct super_block *sb, struct block_private *bp, + int rw) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct bio *bio = NULL; + struct blk_plug plug; + struct page *page; + unsigned long off; + sector_t sector; + int ret = 0; + + sector = bp->bl.blkno << (SCOUTFS_BLOCK_LG_SHIFT - 9); + + WARN_ON_ONCE(bp->bl.blkno == U64_MAX); + WARN_ON_ONCE(sector == U64_MAX || sector == 0); + + /* don't let racing end_io during submission think block is complete */ + atomic_inc(&bp->io_count); + set_bit(BLOCK_BIT_IO_BUSY, &bp->bits); + atomic_inc(&bp->refcount); + + blk_start_plug(&plug); + + for (off = 0; off < SCOUTFS_BLOCK_LG_SIZE; off += PAGE_SIZE) { + if (!bio) { + bio = bio_alloc(GFP_NOFS, SCOUTFS_BLOCK_LG_PAGES_PER); + if (!bio) { + ret = -ENOMEM; + break; + } + + bio->bi_sector = sector + (off >> 9); + bio->bi_bdev = sbi->meta_bdev; + bio->bi_end_io = block_bio_end_io; + bio->bi_private = bp; + + atomic_inc(&bp->io_count); + + TRACE_BLOCK(submit, bp); + } + + if (test_bit(BLOCK_BIT_PAGE_ALLOC, &bp->bits)) + page = virt_to_page((char *)bp->bl.data + off); + else if (test_bit(BLOCK_BIT_VIRT, &bp->bits)) + page = vmalloc_to_page((char *)bp->bl.data + off); + else + BUG(); + + if (!bio_add_page(bio, page, PAGE_SIZE, 0)) { + submit_bio(rw, bio); + bio = NULL; + } + } + + if (bio) + submit_bio(rw, bio); + + blk_finish_plug(&plug); + + /* let racing end_io know we're done */ + block_end_io(sb, rw, bp, ret); + + return ret; +} + +/* + * Return a reference to a cached block in the system, allocating a new + * block if one isn't found in the radix. Its contents are undefined if + * it's newly allocated. + */ +static struct block_private *block_get(struct super_block *sb, u64 blkno) +{ + DECLARE_BLOCK_INFO(sb, binf); + struct block_private *found; + struct block_private *bp; + int ret; + + rcu_read_lock(); + bp = radix_tree_lookup(&binf->radix, blkno); + if (bp) + atomic_inc(&bp->refcount); + rcu_read_unlock(); + + /* drop failed reads that interrupted waiters abandoned */ + if (bp && (test_bit(BLOCK_BIT_ERROR, &bp->bits) && + !test_bit(BLOCK_BIT_DIRTY, &bp->bits))) { + spin_lock(&binf->lock); + block_remove(sb, bp); + spin_unlock(&binf->lock); + block_put(sb, bp); + bp = NULL; + } + + if (!bp) { + bp = block_alloc(sb, blkno); + if (bp == NULL) { + ret = -ENOMEM; + goto out; + } + + ret = radix_tree_preload(GFP_NOFS); + if (ret) + goto out; + + /* could use slot instead of lookup/insert */ + spin_lock(&binf->lock); + found = radix_tree_lookup(&binf->radix, blkno); + if (found) { + atomic_inc(&found->refcount); + } else { + block_insert(sb, bp, blkno); + } + spin_unlock(&binf->lock); + radix_tree_preload_end(); + + if (found) { + block_put(sb, bp); + bp = found; + } + } + + block_accessed(sb, bp); + ret = 0; + +out: + if (ret < 0) { + block_put(sb, bp); + return ERR_PTR(ret); + } + + return bp; +} + +/* + * Return a cached block or a newly allocated block whose contents are + * undefined. The caller is going to initialize the block contents. + */ +struct scoutfs_block *scoutfs_block_create(struct super_block *sb, u64 blkno) +{ + struct block_private *bp; + + bp = block_get(sb, blkno); + if (IS_ERR(bp)) + return ERR_CAST(bp); + + set_bit(BLOCK_BIT_UPTODATE, &bp->bits); + set_bit(BLOCK_BIT_CRC_VALID, &bp->bits); + + return &bp->bl; +} + +static bool uptodate_or_error(struct block_private *bp) +{ + smp_rmb(); /* test after adding to wait queue */ + return test_bit(BLOCK_BIT_UPTODATE, &bp->bits) || + test_bit(BLOCK_BIT_ERROR, &bp->bits); +} + +struct scoutfs_block *scoutfs_block_read(struct super_block *sb, u64 blkno) +{ + DECLARE_BLOCK_INFO(sb, binf); + struct block_private *bp = NULL; + int ret; + + bp = block_get(sb, blkno); + if (IS_ERR(bp)) { + ret = PTR_ERR(bp); + goto out; + } + + if (!test_bit(BLOCK_BIT_UPTODATE, &bp->bits) && + test_and_clear_bit(BLOCK_BIT_NEW, &bp->bits)) { + ret = block_submit_bio(sb, bp, READ); + if (ret < 0) + goto out; + } + + ret = wait_event_interruptible(binf->waitq, uptodate_or_error(bp)); + if (ret == 0 && test_bit(BLOCK_BIT_ERROR, &bp->bits)) + ret = -EIO; + +out: + if (ret < 0) { + block_put(sb, bp); + return ERR_PTR(ret); + } + + return &bp->bl; +} + +/* + * Drop a stale cached read block from the cache. A future read will + * re-read the block from the device. This doesn't drop the caller's reference, + * they still have to call _put. + */ +void scoutfs_block_invalidate(struct super_block *sb, struct scoutfs_block *bl) +{ + DECLARE_BLOCK_INFO(sb, binf); + struct block_private *bp = BLOCK_PRIVATE(bl); + + if (!WARN_ON_ONCE(test_bit(BLOCK_BIT_DIRTY, &bp->bits))) { + scoutfs_inc_counter(sb, block_cache_invalidate); + spin_lock(&binf->lock); + block_remove(sb, bp); + spin_unlock(&binf->lock); + TRACE_BLOCK(invalidate, bp); + } +} + +/* This is only used for large metadata blocks */ +bool scoutfs_block_consistent_ref(struct super_block *sb, + struct scoutfs_block *bl, + __le64 seq, __le64 blkno, u32 magic) +{ + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + struct block_private *bp = BLOCK_PRIVATE(bl); + struct scoutfs_block_header *hdr = bl->data; + + if (!test_bit(BLOCK_BIT_CRC_VALID, &bp->bits)) { + if (hdr->crc != + scoutfs_block_calc_crc(hdr, SCOUTFS_BLOCK_LG_SIZE)) + return false; + set_bit(BLOCK_BIT_CRC_VALID, &bp->bits); + } + + return hdr->magic == cpu_to_le32(magic) && + hdr->fsid == super->hdr.fsid && + hdr->seq == seq && + hdr->blkno == blkno; +} + +void scoutfs_block_put(struct super_block *sb, struct scoutfs_block *bl) +{ + if (!IS_ERR_OR_NULL(bl)) + block_put(sb, BLOCK_PRIVATE(bl)); +} + +void scoutfs_block_writer_init(struct super_block *sb, + struct scoutfs_block_writer *wri) +{ + spin_lock_init(&wri->lock); + INIT_LIST_HEAD(&wri->dirty_list); + wri->nr_dirty_blocks = 0; +} + +/* + * Mark a given block dirty. The caller serializes all dirtying calls + * with writer write calls. As it happens we dirty in allocation order + * and allocate with an advancing cursor so we always dirty in block + * offset order and can walk our list to submit nice ordered IO. + */ +void scoutfs_block_writer_mark_dirty(struct super_block *sb, + struct scoutfs_block_writer *wri, + struct scoutfs_block *bl) +{ + struct block_private *bp = BLOCK_PRIVATE(bl); + + if (!test_and_set_bit(BLOCK_BIT_DIRTY, &bp->bits)) { + BUG_ON(!list_empty(&bp->dirty_entry)); + atomic_inc(&bp->refcount); + spin_lock(&wri->lock); + list_add_tail(&bp->dirty_entry, &wri->dirty_list); + wri->nr_dirty_blocks++; + spin_unlock(&wri->lock); + + TRACE_BLOCK(mark_dirty, bp); + } +} + +bool scoutfs_block_writer_is_dirty(struct super_block *sb, + struct scoutfs_block *bl) +{ + struct block_private *bp = BLOCK_PRIVATE(bl); + + return test_bit(BLOCK_BIT_DIRTY, &bp->bits) != 0; +} + +/* + * Submit writes for all the dirty blocks in the writer's dirty list and + * wait for them to complete. The caller must serialize this with + * attempts to dirty blocks in the writer. If we return an error then + * all the blocks will still be considered dirty. This can be called + * again to attempt to write all the blocks again. + */ +int scoutfs_block_writer_write(struct super_block *sb, + struct scoutfs_block_writer *wri) +{ + DECLARE_BLOCK_INFO(sb, binf); + struct scoutfs_block_header *hdr; + struct block_private *bp; + struct blk_plug plug; + int ret = 0; + + if (wri->nr_dirty_blocks == 0) + return 0; + + /* checksum everything to reduce time between io submission merging */ + list_for_each_entry(bp, &wri->dirty_list, dirty_entry) { + hdr = bp->bl.data; + hdr->crc = scoutfs_block_calc_crc(hdr, SCOUTFS_BLOCK_LG_SIZE); + } + + blk_start_plug(&plug); + + list_for_each_entry(bp, &wri->dirty_list, dirty_entry) { + /* retry previous write errors */ + clear_bit(BLOCK_BIT_ERROR, &bp->bits); + + ret = block_submit_bio(sb, bp, WRITE); + if (ret < 0) + break; + } + + blk_finish_plug(&plug); + + list_for_each_entry(bp, &wri->dirty_list, dirty_entry) { + /* XXX should this be interruptible? */ + wait_event(binf->waitq, !io_busy(bp)); + if (ret == 0 && test_bit(BLOCK_BIT_ERROR, &bp->bits)) { + clear_bit(BLOCK_BIT_ERROR, &bp->bits); + ret = -EIO; + } + } + + if (ret == 0) + scoutfs_block_writer_forget_all(sb, wri); + + return ret; +} + +static void block_forget(struct super_block *sb, + struct scoutfs_block_writer *wri, + struct block_private *bp) +{ + assert_spin_locked(&wri->lock); + + clear_bit(BLOCK_BIT_DIRTY, &bp->bits); + list_del_init(&bp->dirty_entry); + wri->nr_dirty_blocks--; + TRACE_BLOCK(forget, bp); + block_put(sb, bp); +} + +/* + * Clear the dirty status of all the blocks in the writer. The blocks + * remain clean in cache but can be freed by reclaim and then re-read + * from disk, losing whatever modifications made them dirty. + */ +void scoutfs_block_writer_forget_all(struct super_block *sb, + struct scoutfs_block_writer *wri) +{ + struct block_private *tmp; + struct block_private *bp; + + spin_lock(&wri->lock); + + list_for_each_entry_safe(bp, tmp, &wri->dirty_list, dirty_entry) + block_forget(sb, wri, bp); + + spin_unlock(&wri->lock); +} + +/* + * Forget that the given block was dirty. It won't be written in the + * future. Its contents remain in the cache. This is typically used + * as a block is freed. If it is allocated and re-used then its contents + * will be re-initialized. + * + * The caller should ensure that we don't try and mark and forget the + * same block, but this is racing with marking and forgetting other + * blocks. + */ +void scoutfs_block_writer_forget(struct super_block *sb, + struct scoutfs_block_writer *wri, + struct scoutfs_block *bl) +{ + struct block_private *bp = BLOCK_PRIVATE(bl); + + if (test_bit(BLOCK_BIT_DIRTY, &bp->bits)) { + scoutfs_inc_counter(sb, block_cache_forget); + spin_lock(&wri->lock); + if (test_bit(BLOCK_BIT_DIRTY, &bp->bits)) + block_forget(sb, wri, bp); + spin_unlock(&wri->lock); + } +} + +/* + * The caller has ensured that no more dirtying will take place. This + * helps the caller avoid doing a bunch of work before calling into the + * writer to write dirty blocks that didn't exist. + */ +bool scoutfs_block_writer_has_dirty(struct super_block *sb, + struct scoutfs_block_writer *wri) +{ + return wri->nr_dirty_blocks != 0; +} + +/* + * This is a best-effort guess. It's only used for heuristics so it's OK + * if it goes a little bonkers sometimes. + */ +u64 scoutfs_block_writer_dirty_bytes(struct super_block *sb, + struct scoutfs_block_writer *wri) +{ + return wri->nr_dirty_blocks * SCOUTFS_BLOCK_LG_SIZE; +} + +/* + * Remove a number of least recently accessed blocks and free them. We + * don't take locking hit of removing blocks from the lru as they're + * used so this is racing with accesses holding an elevated refcount. + * We check the refcount to attempt to not free a block that snuck in + * and is being accessed while the block is still at the head of the + * LRU. + * + * Dirty blocks will always have an elevated refcount (and will be + * likely be towards the tail of the LRU). Even if we do remove them + * from the LRU their dirty refcount will keep them live until IO + * completes and their dirty refcount is dropped. + */ +static int block_shrink(struct shrinker *shrink, struct shrink_control *sc) +{ + struct block_info *binf = container_of(shrink, struct block_info, + shrinker); + struct super_block *sb = binf->sb; + struct block_private *tmp; + struct block_private *bp; + unsigned long nr; + LIST_HEAD(list); + + nr = sc->nr_to_scan; + if (!nr) + goto out; + + spin_lock(&binf->lock); + + list_for_each_entry_safe(bp, tmp, &binf->lru_list, lru_entry) { + + if (atomic_read(&bp->refcount) > 1) + continue; + + if (nr-- == 0) + break; + + TRACE_BLOCK(shrink, bp); + + scoutfs_inc_counter(sb, block_cache_shrink); + block_remove(sb, bp); + + } + + spin_unlock(&binf->lock); + +out: + return min_t(u64, binf->lru_nr * SCOUTFS_BLOCK_LG_PAGES_PER, INT_MAX); +} + +struct sm_block_completion { + struct completion comp; + int err; +}; + +static void sm_block_bio_end_io(struct bio *bio, int err) +{ + struct sm_block_completion *sbc = bio->bi_private; + + sbc->err = err; + complete(&sbc->comp); + bio_put(bio); +} + +/* + * Perform a private synchronous read or write of a small fixed size 4K + * block. We allocate a private page and bio and copy to or from the + * caller's buffer. + * + * The interface is a little weird because our blocks always start with + * a block header that contains a crc of the entire block. We're the + * only layer that sees the full block buffer so we pass the calculated + * crc to the caller for them to check in their context. + */ +static int sm_block_io(struct block_device *bdev, int rw, u64 blkno, + struct scoutfs_block_header *hdr, size_t len, + __le32 *blk_crc) +{ + struct scoutfs_block_header *pg_hdr; + struct sm_block_completion sbc; + struct page *page; + struct bio *bio; + int ret; + + BUILD_BUG_ON(PAGE_SIZE < SCOUTFS_BLOCK_SM_SIZE); + + if (WARN_ON_ONCE(len > SCOUTFS_BLOCK_SM_SIZE) || + WARN_ON_ONCE(!(rw & WRITE) && !blk_crc)) + return -EINVAL; + + page = alloc_page(GFP_NOFS); + if (!page) + return -ENOMEM; + + pg_hdr = page_address(page); + + if (rw & WRITE) { + memcpy(pg_hdr, hdr, len); + if (len < SCOUTFS_BLOCK_SM_SIZE) + memset((char *)pg_hdr + len, 0, + SCOUTFS_BLOCK_SM_SIZE - len); + pg_hdr->crc = scoutfs_block_calc_crc(pg_hdr, + SCOUTFS_BLOCK_SM_SIZE); + } + + bio = bio_alloc(GFP_NOFS, 1); + if (!bio) { + ret = -ENOMEM; + goto out; + } + + bio->bi_sector = blkno << (SCOUTFS_BLOCK_SM_SHIFT - 9); + bio->bi_bdev = bdev; + bio->bi_end_io = sm_block_bio_end_io; + bio->bi_private = &sbc; + bio_add_page(bio, page, SCOUTFS_BLOCK_SM_SIZE, 0); + + init_completion(&sbc.comp); + sbc.err = 0; + + submit_bio((rw & WRITE) ? WRITE_SYNC : READ_SYNC, bio); + + wait_for_completion(&sbc.comp); + ret = sbc.err; + + if (ret == 0 && !(rw & WRITE)) { + memcpy(hdr, pg_hdr, len); + *blk_crc = scoutfs_block_calc_crc(pg_hdr, + SCOUTFS_BLOCK_SM_SIZE); + } +out: + __free_page(page); + return ret; +} + +int scoutfs_block_read_sm(struct super_block *sb, + struct block_device *bdev, u64 blkno, + struct scoutfs_block_header *hdr, size_t len, + __le32 *blk_crc) +{ + return sm_block_io(bdev, READ, blkno, hdr, len, blk_crc); +} + +int scoutfs_block_write_sm(struct super_block *sb, + struct block_device *bdev, u64 blkno, + struct scoutfs_block_header *hdr, size_t len) +{ + return sm_block_io(bdev, WRITE, blkno, hdr, len, NULL); +} + +int scoutfs_block_setup(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct block_info *binf; + loff_t size; + int ret; + + /* we store blknos in longs in the radix */ + size = i_size_read(sb->s_bdev->bd_inode); + if ((size >> SCOUTFS_BLOCK_LG_SHIFT) >= LONG_MAX) { + scoutfs_err(sb, "Cant reference all blocks in %llu byte device with %u bit long radix tree indexes", + size, BITS_PER_LONG); + return -EINVAL; + } + + binf = kzalloc(sizeof(struct block_info), GFP_KERNEL); + if (!binf) { + ret = -ENOMEM; + goto out; + } + + binf->sb = sb; + spin_lock_init(&binf->lock); + INIT_RADIX_TREE(&binf->radix, GFP_ATOMIC); /* insertion preloads */ + INIT_LIST_HEAD(&binf->lru_list); + init_waitqueue_head(&binf->waitq); + binf->shrinker.shrink = block_shrink; + binf->shrinker.seeks = DEFAULT_SEEKS; + register_shrinker(&binf->shrinker); + INIT_WORK(&binf->free_work, block_free_work); + init_llist_head(&binf->free_llist); + + sbi->block_info = binf; + + ret = 0; +out: + if (ret) + scoutfs_block_destroy(sb); + + return 0; +} + +void scoutfs_block_destroy(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct block_info *binf = SCOUTFS_SB(sb)->block_info; + + if (binf) { + unregister_shrinker(&binf->shrinker); + block_remove_all(sb); + flush_work(&binf->free_work); + + WARN_ON_ONCE(!llist_empty(&binf->free_llist)); + kfree(binf); + + sbi->block_info = NULL; + } +} diff --git a/kmod/src/block.h b/kmod/src/block.h new file mode 100644 index 00000000..79a859d7 --- /dev/null +++ b/kmod/src/block.h @@ -0,0 +1,60 @@ +#ifndef _SCOUTFS_BLOCK_H_ +#define _SCOUTFS_BLOCK_H_ + +struct scoutfs_block_writer { + spinlock_t lock; + struct list_head dirty_list; + u64 nr_dirty_blocks; +}; + +struct scoutfs_block { + u64 blkno; + void *data; + void *priv; +}; + +__le32 scoutfs_block_calc_crc(struct scoutfs_block_header *hdr, u32 size); +bool scoutfs_block_valid_crc(struct scoutfs_block_header *hdr, u32 size); +bool scoutfs_block_valid_ref(struct super_block *sb, + struct scoutfs_block_header *hdr, + __le64 seq, __le64 blkno); + +struct scoutfs_block *scoutfs_block_create(struct super_block *sb, u64 blkno); +struct scoutfs_block *scoutfs_block_read(struct super_block *sb, u64 blkno); +void scoutfs_block_invalidate(struct super_block *sb, struct scoutfs_block *bl); +bool scoutfs_block_consistent_ref(struct super_block *sb, + struct scoutfs_block *bl, + __le64 seq, __le64 blkno, u32 magic); +void scoutfs_block_put(struct super_block *sb, struct scoutfs_block *bl); + +void scoutfs_block_writer_init(struct super_block *sb, + struct scoutfs_block_writer *wri); +void scoutfs_block_writer_mark_dirty(struct super_block *sb, + struct scoutfs_block_writer *wri, + struct scoutfs_block *bl); +bool scoutfs_block_writer_is_dirty(struct super_block *sb, + struct scoutfs_block *bl); +int scoutfs_block_writer_write(struct super_block *sb, + struct scoutfs_block_writer *wri); +void scoutfs_block_writer_forget_all(struct super_block *sb, + struct scoutfs_block_writer *wri); +void scoutfs_block_writer_forget(struct super_block *sb, + struct scoutfs_block_writer *wri, + struct scoutfs_block *bl); +bool scoutfs_block_writer_has_dirty(struct super_block *sb, + struct scoutfs_block_writer *wri); +u64 scoutfs_block_writer_dirty_bytes(struct super_block *sb, + struct scoutfs_block_writer *wri); + +int scoutfs_block_read_sm(struct super_block *sb, + struct block_device *bdev, u64 blkno, + struct scoutfs_block_header *hdr, size_t len, + __le32 *blk_crc); +int scoutfs_block_write_sm(struct super_block *sb, + struct block_device *bdev, u64 blkno, + struct scoutfs_block_header *hdr, size_t len); + +int scoutfs_block_setup(struct super_block *sb); +void scoutfs_block_destroy(struct super_block *sb); + +#endif diff --git a/kmod/src/btree.c b/kmod/src/btree.c new file mode 100644 index 00000000..d4eee1cc --- /dev/null +++ b/kmod/src/btree.c @@ -0,0 +1,1842 @@ +/* + * Copyright (C) 2017 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include +#include +#include +#include + +#include "super.h" +#include "format.h" +#include "key.h" +#include "btree.h" +#include "counters.h" +#include "triggers.h" +#include "options.h" +#include "msg.h" +#include "block.h" +#include "alloc.h" +#include "avl.h" +#include "hash.h" +#include "sort_priv.h" + +#include "scoutfs_trace.h" + +/* + * scoutfs uses a cow btree to index fs metadata. + * + * Today callers provide all the locking. They serialize readers and + * writers and writers and committing all the dirty blocks. + * + * Block reference have sufficient metadata to discover corrupt + * references. If a reader encounters a bad block it backs off which + * gives the caller the opportunity to resample the root in case it was + * reading through a stale btree that has been overwritten. This lets + * mounts read trees that are modified by other mounts without exclusive + * locking. + * + * Btree items are stored as a dense array of structs at the front of + * each block. New items are allocated at the end of the array. + * Deleted items are swapped with the last item to maintain the dense + * array. The items are indexed by a balanced binary tree with parent + * pointers so the relocated item can have references to it updated. + * + * Values are allocated from the end of the block towards the front, + * consuming the end of free space in the center of the block. Deleted + * values create fragmented free space in other existing values. Rather + * than tracking free space specifically, we compact values in bulk to + * defragment free space if there is enough of to be worth the cost of + * compaction. When there's only a little bit of fragmented free space + * we split the block as usual. + * + * Exact item searches are only performed on leaf blocks. Leaf blocks + * have a hash table at the end of the block which is used to find items + * with a specific key. It uses linear probing and maintains a low load + * factor so any given search will most likely only need a single + * cacheline. + * + * Parent block reference items are stored as items with a block + * reference as a value. There's an item with a key for every child + * reference instead of having separator keys between child references. + * The key in a child reference contains the largest key that may be + * found in the child subtree. The right spine of the tree has maximal + * keys so that they don't have to be updated if we insert an item with + * a key greater than everything in the tree. + */ + +/* btree walking has a bunch of behavioural bit flags */ +enum btree_walk_flags { + BTW_NEXT = (1 << 0), /* return >= key */ + BTW_PREV = (1 << 1), /* return <= key */ + BTW_DIRTY = (1 << 2), /* cow stable blocks */ + BTW_ALLOC = (1 << 3), /* allocate a new block for 0 ref */ + BTW_INSERT = (1 << 4), /* walking to insert, try splitting */ + BTW_DELETE = (1 << 5), /* walking to delete, try joining */ +}; + +/* total length of the value payload */ +static inline unsigned int val_bytes(unsigned val_len) +{ + return round_up(val_len, SCOUTFS_BTREE_VALUE_ALIGN); +} + +/* number of bytes in a block used by an item with the given value length */ +static inline unsigned int item_len_bytes(unsigned val_len) +{ + return sizeof(struct scoutfs_btree_item) + val_bytes(val_len); +} + +/* number of bytes used by an existing item */ +static inline unsigned int item_bytes(struct scoutfs_btree_item *item) +{ + return item_len_bytes(le16_to_cpu(item->val_len)); +} + +/* + * Join blocks when they both are 1/4 full. This puts some distance + * between the join threshold and the full threshold for splitting. + * Blocks that just split or joined need to undergo a reasonable amount + * of item modification before they'll split or join again. + */ +static unsigned int join_low_watermark(void) +{ + return (SCOUTFS_BLOCK_LG_SIZE - sizeof(struct scoutfs_btree_block)) / 4; +} + +/* + * return the integer percentages of total space the block could have + * consumed by items that is currently consumed. + */ +static unsigned int item_full_pct(struct scoutfs_btree_block *bt) +{ + return (int)le16_to_cpu(bt->total_item_bytes) * 100 / + (SCOUTFS_BLOCK_LG_SIZE - sizeof(struct scoutfs_btree_block)); +} + +static inline __le16 ptr_off(struct scoutfs_btree_block *bt, void *ptr) +{ + return cpu_to_le16(ptr - (void *)bt); +} + +static inline void *off_ptr(struct scoutfs_btree_block *bt, u16 off) +{ + return (void *)bt + off; +} + +static inline struct scoutfs_btree_item * +off_item(struct scoutfs_btree_block *bt, __le16 off) +{ + return (void *)bt + le16_to_cpu(off); +} + + +/* + * The item at the end of the item array. This is *not* the item in the + * block with the greatest key. + */ +static struct scoutfs_btree_item *end_item(struct scoutfs_btree_block *bt) +{ + BUG_ON(bt->nr_items == 0); + + return &bt->items[le16_to_cpu(bt->nr_items) - 1]; +} + +/* offset of the start of the free range in the middle of the block */ +static inline unsigned int mid_free_off(struct scoutfs_btree_block *bt) +{ + return le16_to_cpu(ptr_off(bt, &bt->items[le16_to_cpu(bt->nr_items)])); +} + +/* true if the mid free region has room for an item struct and its value */ +static inline bool mid_free_item_room(struct scoutfs_btree_block *bt, + int val_len) +{ + return le16_to_cpu(bt->mid_free_len) >= item_len_bytes(val_len); +} + +static inline struct scoutfs_key *item_key(struct scoutfs_btree_item *item) +{ + return &item->key; +} + +static inline void *item_val(struct scoutfs_btree_block *bt, + struct scoutfs_btree_item *item) +{ + return off_ptr(bt, le16_to_cpu(item->val_off)); +} + +static inline unsigned item_val_len(struct scoutfs_btree_item *item) +{ + return le16_to_cpu(item->val_len); +} + +static struct scoutfs_btree_item *node_item(struct scoutfs_avl_node *node) +{ + if (node == NULL) + return NULL; + return container_of(node, struct scoutfs_btree_item, node); +} + +static struct scoutfs_btree_item *last_item(struct scoutfs_btree_block *bt) +{ + return node_item(scoutfs_avl_last(&bt->item_root)); +} + +static struct scoutfs_btree_item *prev_item(struct scoutfs_btree_block *bt, + struct scoutfs_btree_item *item) +{ + if (item == NULL) + return NULL; + return node_item(scoutfs_avl_prev(&bt->item_root, &item->node)); +} + +static struct scoutfs_btree_item *next_item(struct scoutfs_btree_block *bt, + struct scoutfs_btree_item *item) +{ + if (item == NULL) + return NULL; + return node_item(scoutfs_avl_next(&bt->item_root, &item->node)); +} + +static int cmp_key_item(void *arg, struct scoutfs_avl_node *node) +{ + struct scoutfs_key *key = arg; + struct scoutfs_btree_item *item = node_item(node); + + return scoutfs_key_compare(key, item_key(item)); +} + +/* + * We have a small fixed-size linearly probed hash table at the end of + * leaf blocks which is used for direct item lookups (as opposed to + * iterators). The hash table only stores non-zero offsets to the + * items. If an item is moved then its offset is updated. The hash + * table is sized to allow a max load of 75%, but most items are larger + * and most blocks aren't full. + */ +static int leaf_item_hash_ind(struct scoutfs_key *key) +{ + return scoutfs_hash32(key, sizeof(struct scoutfs_key)) % + SCOUTFS_BTREE_LEAF_ITEM_HASH_NR; +} + +static __le16 *leaf_item_hash_buckets(struct scoutfs_btree_block *bt) +{ + return (void *)bt + SCOUTFS_BLOCK_LG_SIZE - + SCOUTFS_BTREE_LEAF_ITEM_HASH_BYTES; +} + +static inline int leaf_item_hash_next_bucket(int i) +{ + if (++i >= SCOUTFS_BTREE_LEAF_ITEM_HASH_NR) + i = 0; + return i; +} + +#define foreach_leaf_item_hash_bucket(i, nr, key) \ + for (i = leaf_item_hash_ind(key), nr = SCOUTFS_BTREE_LEAF_ITEM_HASH_NR;\ + nr-- > 0; \ + i = leaf_item_hash_next_bucket(i)) + +static struct scoutfs_btree_item * +leaf_item_hash_search(struct super_block *sb, struct scoutfs_btree_block *bt, + struct scoutfs_key *key) +{ + __le16 *buckets = leaf_item_hash_buckets(bt); + struct scoutfs_btree_item *item; + __le16 off; + int nr; + int i; + + scoutfs_inc_counter(sb, btree_leaf_item_hash_search); + + if (WARN_ON_ONCE(bt->level > 0)) + return NULL; + + foreach_leaf_item_hash_bucket(i, nr, key) { + off = buckets[i]; + if (off == 0) + return NULL; + + item = off_item(bt, off); + if (scoutfs_key_compare(key, item_key(item)) == 0) + return item; + } + + return NULL; +} + +static void leaf_item_hash_insert(struct scoutfs_btree_block *bt, + struct scoutfs_key *key, __le16 off) +{ + __le16 *buckets = leaf_item_hash_buckets(bt); + int nr; + int i; + + if (bt->level > 0) + return; + + foreach_leaf_item_hash_bucket(i, nr, key) { + if (buckets[i] == 0) { + buckets[i] = off; + return; + } + } + + /* table should have been been enough for all items */ + BUG(); +} + +/* + * Deletion clears the offset in a bucket. That could create a + * discontinuity that would stop a search from seeing colliding + * insertions that were pushed into further buckets. Each time we zero + * a bucket we rehash all the populated buckets following it. There + * won't be many in our light load tables and this works reliably as the + * contiguous population wraps past the end of table. Comparing hashed + * bucket positions to find candidates to relocate after the wrap is + * tricky. + */ +static void leaf_item_hash_delete(struct scoutfs_btree_block *bt, + struct scoutfs_key *key, __le16 del_off) +{ + __le16 *buckets = leaf_item_hash_buckets(bt); + __le16 off; + int nr; + int i; + + if (bt->level > 0) + return; + + foreach_leaf_item_hash_bucket(i, nr, key) { + off = buckets[i]; + /* we must find the item we're trying to delete */ + BUG_ON(off == 0); + + if (off == del_off) { + buckets[i] = 0; + break; + } + } + + while ((i = leaf_item_hash_next_bucket(i)), buckets[i] != 0) { + off = buckets[i]; + buckets[i] = 0; + leaf_item_hash_insert(bt, item_key(off_item(bt, off)), off); + } +} + +static void leaf_item_hash_change(struct scoutfs_btree_block *bt, + struct scoutfs_key *key, __le16 to, + __le16 from) +{ + __le16 *buckets = leaf_item_hash_buckets(bt); + __le16 off; + int nr; + int i; + + if (bt->level > 0) + return; + + foreach_leaf_item_hash_bucket(i, nr, key) { + off = buckets[i]; + /* we must find the item we're trying to change */ + BUG_ON(off == 0); + + if (off == from) { + buckets[i] = to; + return; + } + } +} + +static int cmp_sorted(void *priv, const void *A, const void *B) +{ + struct scoutfs_btree_block *bt = priv; + const unsigned short *a = A; + const unsigned short *b = B; + struct scoutfs_btree_item *item_a = &bt->items[*a]; + struct scoutfs_btree_item *item_b = &bt->items[*b]; + + return scoutfs_cmp(le16_to_cpu(item_a->val_off), + le16_to_cpu(item_b->val_off)); +} + +static void swap_sorted(void *priv, void *A, void *B, int size) +{ + unsigned short *a = A; + unsigned short *b = B; + + swap(*a, *b); +} + +/* + * As values are freed they can leave fragmented free space amongst + * other values. We compact the values by sorting an array of item + * indices by the offset of the item's values. We can then walk values + * from the back of the block and pack them into contiguous space, + * bubbling any fragmented free space towards the middle. + * + * This is called when we can't insert because there isn't enough + * available free space in the middle of the block but we know that + * there's sufficient free fragmented space in the values. + * + * We only want to compact when there is enough free space to justify + * the cost of the compaction. We don't want to bother compacting if + * the block is almost full and we just be split in a few more + * operations. The split heuristic requires a generous amount of + * fragmented free space that will avoid a split. + */ +static int compact_values(struct super_block *sb, + struct scoutfs_btree_block *bt) +{ + const int nr = le16_to_cpu(bt->nr_items); + struct scoutfs_btree_item *item; + unsigned short *sorted = NULL; + unsigned int to_off; + unsigned int vb; + void *from; + void *to; + int i; + + scoutfs_inc_counter(sb, btree_compact_values); + + BUILD_BUG_ON(sizeof(sorted[0]) != sizeof(bt->nr_items)); + + sorted = kmalloc_array(le16_to_cpu(bt->nr_items), sizeof(sorted[0]), + GFP_NOFS); + if (!sorted) { + scoutfs_inc_counter(sb, btree_compact_values_enomem); + return -ENOMEM; + } + + /* sort the sorted array of item indices by their value offset */ + for (i = 0; i < nr; i++) + sorted[i] = i; + sort_priv(bt, sorted, nr, sizeof(sorted[0]), cmp_sorted, swap_sorted); + + to_off = SCOUTFS_BLOCK_LG_SIZE; + if (bt->level == 0) + to_off -= SCOUTFS_BTREE_LEAF_ITEM_HASH_BYTES; + + /* move values towards the back of the block */ + for (i = nr - 1; i >= 0; i--) { + item = &bt->items[sorted[i]]; + if (item->val_len == 0) + continue; + + vb = val_bytes(le16_to_cpu(item->val_len)); + to_off -= vb; + from = off_ptr(bt, le16_to_cpu(item->val_off)); + to = off_ptr(bt, to_off); + + if (from != to) { + if (to >= from + vb) + memcpy(to, from, vb); + else + memmove(to, from, vb); + + item->val_off = cpu_to_le16(to_off); + } + } + + bt->mid_free_len = cpu_to_le16(to_off - mid_free_off(bt)); + + kfree(sorted); + return 0; +} + +/* + * Insert an item's value into the block. The caller has made sure + * there's free space. We store the value at the end of free space in + * the block and point its final offset at its owning item, and copy the + * value into place. + */ +static __le16 insert_value(struct scoutfs_btree_block *bt, __le16 item_off, + void *val, unsigned val_len) +{ + unsigned int val_off; + unsigned int vb; + + if (val_len == 0) + return 0; + + BUG_ON(le16_to_cpu(bt->mid_free_len) < val_bytes(val_len)); + + vb = val_bytes(val_len); + val_off = mid_free_off(bt) + le16_to_cpu(bt->mid_free_len) - vb; + le16_add_cpu(&bt->mid_free_len, -vb); + + memcpy(off_ptr(bt, val_off), val, val_len); + + return cpu_to_le16(val_off); +} + +/* + * Insert a new item into the block. The caller has made sure that + * there is sufficient free space in block for the new item. We might + * have to compact the values to the end of the block to reclaim + * fragmented free space between values. + * + * This only consumes free space. It's safe to use references to block + * structures after this call. + */ +static void create_item(struct scoutfs_btree_block *bt, + struct scoutfs_key *key, void *val, unsigned val_len, + struct scoutfs_avl_node *parent, int cmp) +{ + struct scoutfs_btree_item *item; + + BUG_ON(le16_to_cpu(bt->mid_free_len) < item_len_bytes(val_len)); + + le16_add_cpu(&bt->mid_free_len, + -(u16)sizeof(struct scoutfs_btree_item)); + le16_add_cpu(&bt->nr_items, 1); + item = end_item(bt); + + item->key = *key; + + scoutfs_avl_insert(&bt->item_root, parent, &item->node, cmp); + leaf_item_hash_insert(bt, item_key(item), ptr_off(bt, item)); + + item->val_off = insert_value(bt, ptr_off(bt, item), val, val_len); + item->val_len = cpu_to_le16(val_len); + + le16_add_cpu(&bt->total_item_bytes, item_bytes(item)); +} + +/* + * Delete an item from a btree block. + * + * As we delete the item we can relocate an unrelated item to maintain + * the dense array of items. The caller can use another single item + * after this call if they give us the opportunity to let them know if + * we move it. + */ +static void delete_item(struct scoutfs_btree_block *bt, + struct scoutfs_btree_item *item, + struct scoutfs_btree_item **use_after) +{ + struct scoutfs_btree_item *end; + unsigned int val_off; + unsigned int val_len; + + /* save some values before we delete the item */ + val_off = le16_to_cpu(item->val_off); + val_len = le16_to_cpu(item->val_len); + end = end_item(bt); + + /* delete the item */ + scoutfs_avl_delete(&bt->item_root, &item->node); + leaf_item_hash_delete(bt, item_key(item), ptr_off(bt, item)); + le16_add_cpu(&bt->nr_items, -1); + le16_add_cpu(&bt->mid_free_len, sizeof(struct scoutfs_btree_item)); + le16_add_cpu(&bt->total_item_bytes, -item_bytes(item)); + + /* move the final item into the deleted space */ + if (end != item) { + item->key = end->key; + item->val_off = end->val_off; + item->val_len = end->val_len; + leaf_item_hash_change(bt, &end->key, ptr_off(bt, item), + ptr_off(bt, end)); + scoutfs_avl_relocate(&bt->item_root, &item->node,&end->node); + if (use_after && *use_after == end) + *use_after = item; + } +} + +/* + * Move items from a source block to a destination block. The caller + * has made sure there's sufficient free space in the destination block, + * though item creation may need to compact values. The caller tells us + * if we're moving from the tail of the source block right to the head + * of the destination block, or vice versa. We're always adding the + * first or last item to the avl, so the parent is always the previous + * first or last node. + */ +static void move_items(struct scoutfs_btree_block *dst, + struct scoutfs_btree_block *src, bool move_right, + int to_move) +{ + struct scoutfs_avl_node *par; + struct scoutfs_avl_node *node; + struct scoutfs_btree_item *from; + struct scoutfs_btree_item *next; + int cmp; + + if (move_right) { + node = scoutfs_avl_last(&src->item_root); + par = scoutfs_avl_first(&dst->item_root); + cmp = -1; + } else { + node = scoutfs_avl_first(&src->item_root); + par = scoutfs_avl_last(&dst->item_root); + cmp = 1; + } + from = node_item(node); + + while (to_move > 0 && from != NULL) { + to_move -= item_bytes(from); + + if (move_right) + next = prev_item(src, from); + else + next = next_item(src, from); + + create_item(dst, item_key(from), item_val(src, from), + item_val_len(from), par, cmp); + + if (move_right) { + if (par) + par = scoutfs_avl_prev(&dst->item_root, par); + else + par = scoutfs_avl_first(&dst->item_root); + } else { + if (par) + par = scoutfs_avl_next(&dst->item_root, par); + else + par = scoutfs_avl_last(&dst->item_root); + } + + delete_item(src, from, &next); + from = next; + } +} + +/* + * This is used to lookup cached blocks, read blocks, cow blocks for + * dirtying, and allocate new blocks. + * + * Btree blocks don't have rigid cache consistency. We can be following + * block references into cached blocks that are now stale or can be + * following a stale root into blocks that have been overwritten. If we + * hit a block that looks stale we first invalidate the cache and retry, + * returning -ESTALE if it still looks wrong. The caller can retry the + * read from a more current root or decide that this is a persistent + * error. + */ +static int get_ref_block(struct super_block *sb, + struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, int flags, + struct scoutfs_btree_ref *ref, + struct scoutfs_block **bl_ret) +{ + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + struct scoutfs_btree_block *bt = NULL; + struct scoutfs_btree_block *new; + struct scoutfs_block *new_bl = NULL; + struct scoutfs_block *bl = NULL; + bool retried = false; + u64 blkno; + u64 seq; + int ret; + + /* always get the current block, either to return or cow from */ + if (ref && ref->blkno) { +retry: + + bl = scoutfs_block_read(sb, le64_to_cpu(ref->blkno)); + if (IS_ERR(bl)) { + trace_scoutfs_btree_read_error(sb, ref); + scoutfs_inc_counter(sb, btree_read_error); + ret = PTR_ERR(bl); + goto out; + } + bt = (void *)bl->data; + + if (!scoutfs_block_consistent_ref(sb, bl, ref->seq, ref->blkno, + SCOUTFS_BLOCK_MAGIC_BTREE) || + scoutfs_trigger(sb, BTREE_STALE_READ)) { + + scoutfs_inc_counter(sb, btree_stale_read); + + scoutfs_block_invalidate(sb, bl); + scoutfs_block_put(sb, bl); + bl = NULL; + + if (!retried) { + retried = true; + goto retry; + } + + ret = -ESTALE; + goto out; + } + + /* + * We need to create a new dirty copy of the block if + * the caller asked for it. If the block is already + * dirty then we can return it. + */ + if (!(flags & BTW_DIRTY) || + scoutfs_block_writer_is_dirty(sb, bl)) { + ret = 0; + goto out; + } + + } else if (!(flags & BTW_ALLOC)) { + ret = -ENOENT; + goto out; + } + + ret = scoutfs_alloc_meta(sb, alloc, wri, &blkno); + if (ret < 0) + goto out; + + prandom_bytes(&seq, sizeof(seq)); + + new_bl = scoutfs_block_create(sb, blkno); + if (IS_ERR(new_bl)) { + ret = scoutfs_free_meta(sb, alloc, wri, blkno); + BUG_ON(ret); + ret = PTR_ERR(new_bl); + goto out; + } + new = (void *)new_bl->data; + + /* free old stable blkno we're about to overwrite */ + if (ref && ref->blkno) { + ret = scoutfs_free_meta(sb, alloc, wri, + le64_to_cpu(ref->blkno)); + if (ret) { + ret = scoutfs_free_meta(sb, alloc, wri, blkno); + BUG_ON(ret); + scoutfs_block_put(sb, new_bl); + new_bl = NULL; + goto out; + } + } + + scoutfs_block_writer_mark_dirty(sb, wri, new_bl); + + trace_scoutfs_btree_dirty_block(sb, blkno, seq, + bt ? le64_to_cpu(bt->hdr.blkno) : 0, + bt ? le64_to_cpu(bt->hdr.seq) : 0); + + if (bt) { + /* returning a cow of an existing block */ + memcpy(new, bt, SCOUTFS_BLOCK_LG_SIZE); + scoutfs_block_put(sb, bl); + } else { + /* returning a newly allocated block */ + memset(new, 0, SCOUTFS_BLOCK_LG_SIZE); + new->hdr.fsid = super->hdr.fsid; + } + bl = new_bl; + bt = new; + + bt->hdr.magic = cpu_to_le32(SCOUTFS_BLOCK_MAGIC_BTREE); + bt->hdr.blkno = cpu_to_le64(blkno); + bt->hdr.seq = cpu_to_le64(seq); + if (ref) { + ref->blkno = bt->hdr.blkno; + ref->seq = bt->hdr.seq; + } + ret = 0; + +out: + if (ret) { + scoutfs_block_put(sb, bl); + bl = NULL; + } + + *bl_ret = bl; + return ret; +} + +/* + * Create a new item in the parent which references the child. The caller + * specifies the key in the item that describes the items in the child. + */ +static void create_parent_item(struct scoutfs_btree_block *parent, + struct scoutfs_btree_block *child, + struct scoutfs_key *key) +{ + struct scoutfs_avl_node *par; + int cmp; + struct scoutfs_btree_ref ref = { + .blkno = child->hdr.blkno, + .seq = child->hdr.seq, + }; + + scoutfs_avl_search(&parent->item_root, cmp_key_item, key, &cmp, &par, + NULL, NULL); + create_item(parent, key, &ref, sizeof(ref), par, cmp); +} + +/* + * Update an existing parent item reference to a child who may be new or + * may have had its last item changed. + */ +static void update_parent_item(struct scoutfs_btree_block *parent, + struct scoutfs_btree_item *par_item, + struct scoutfs_btree_block *child) +{ + struct scoutfs_btree_ref *ref = item_val(parent, par_item); + + par_item->key = *item_key(last_item(child)); + ref->blkno = child->hdr.blkno; + ref->seq = child->hdr.seq; +} + +static __le16 init_mid_free_len(int level) +{ + int free; + + free = SCOUTFS_BLOCK_LG_SIZE - sizeof(struct scoutfs_btree_block); + if (level == 0) + free -= SCOUTFS_BTREE_LEAF_ITEM_HASH_BYTES; + + return cpu_to_le16(free); +} + +static void init_btree_block(struct scoutfs_btree_block *bt, int level) +{ + + bt->level = level; + bt->mid_free_len = init_mid_free_len(level); +} + +/* + * See if we need to split this block while descending for insertion so + * that we have enough space to insert. Parent blocks need enough space + * to insert a new parent item if a child block splits. Leaf blocks + * need enough space to insert the new item with its value. + * + * We split to the left so that the greatest key in the existing block + * doesn't change so we don't have to update the key in its parent item. + * + * Returns -errno, 0 if nothing done, or 1 if we split. + */ +static int try_split(struct super_block *sb, + struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_btree_root *root, + struct scoutfs_key *key, unsigned val_len, + struct scoutfs_btree_block *parent, + struct scoutfs_btree_block *right) +{ + struct scoutfs_block *left_bl = NULL; + struct scoutfs_block *par_bl = NULL; + struct scoutfs_btree_block *left; + struct scoutfs_key max_key; + int ret; + int err; + + /* parents need to leave room for child references */ + if (right->level) + val_len = sizeof(struct scoutfs_btree_ref); + + /* don't need to split if there's enough space for the item */ + if (mid_free_item_room(right, val_len)) + return 0; + + if (item_full_pct(right) < 80) + return compact_values(sb, right); + + scoutfs_inc_counter(sb, btree_split); + + /* alloc split neighbour first to avoid unwinding tree growth */ + ret = get_ref_block(sb, alloc, wri, BTW_ALLOC, NULL, &left_bl); + if (ret) + return ret; + left = left_bl->data; + + init_btree_block(left, right->level); + + if (!parent) { + ret = get_ref_block(sb, alloc, wri, BTW_ALLOC, NULL, &par_bl); + if (ret) { + err = scoutfs_free_meta(sb, alloc, wri, + le64_to_cpu(left->hdr.blkno)); + BUG_ON(err); /* radix should have been dirty */ + scoutfs_block_put(sb, left_bl); + return ret; + } + parent = par_bl->data; + + init_btree_block(parent, root->height); + root->height++; + root->ref.blkno = parent->hdr.blkno; + root->ref.seq = parent->hdr.seq; + + scoutfs_key_set_ones(&max_key); + create_parent_item(parent, right, &max_key); + } + + move_items(left, right, false, + le16_to_cpu(right->total_item_bytes) / 2); + + create_parent_item(parent, left, item_key(last_item(left))); + + scoutfs_block_put(sb, left_bl); + scoutfs_block_put(sb, par_bl); + + return 1; +} + +/* + * This is called during descent for deletion when we have a parent and + * might need to join this block with a sibling block if this block has + * too much free space. Eventually we'll be able to fit all of the + * sibling's items in our free space which lets us delete the sibling + * block. + */ +static int try_join(struct super_block *sb, + struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_btree_root *root, + struct scoutfs_btree_block *parent, + struct scoutfs_btree_item *par_item, + struct scoutfs_btree_block *bt) +{ + struct scoutfs_btree_item *sib_par_item; + struct scoutfs_btree_block *sib; + struct scoutfs_block *sib_bl; + struct scoutfs_btree_ref *ref; + unsigned int sib_tot; + bool move_right; + int to_move; + int ret; + + if (le16_to_cpu(bt->total_item_bytes) >= join_low_watermark()) + return 0; + + scoutfs_inc_counter(sb, btree_join); + + /* move items right into our block if we have a left sibling */ + sib_par_item = prev_item(parent, par_item); + if (sib_par_item) { + move_right = true; + } else { + sib_par_item = next_item(parent, par_item); + move_right = false; + } + + ref = item_val(parent, sib_par_item); + ret = get_ref_block(sb, alloc, wri, BTW_DIRTY, ref, &sib_bl); + if (ret) + return ret; + sib = sib_bl->data; + + sib_tot = le16_to_cpu(bt->total_item_bytes); + if (sib_tot < join_low_watermark()) + to_move = sib_tot; + else + to_move = sib_tot - join_low_watermark(); + + if (le16_to_cpu(bt->mid_free_len) < to_move) { + ret = compact_values(sb, bt); + if (ret < 0) + scoutfs_block_put(sb, sib_bl); + return ret; + } + move_items(bt, sib, move_right, to_move); + + /* update our parent's item */ + if (!move_right) + update_parent_item(parent, par_item, bt); + + /* update or delete sibling's parent item */ + if (le16_to_cpu(sib->nr_items) == 0) { + delete_item(parent, sib_par_item, NULL); + ret = scoutfs_free_meta(sb, alloc, wri, + le64_to_cpu(sib->hdr.blkno)); + BUG_ON(ret); + + } else if (move_right) { + update_parent_item(parent, sib_par_item, sib); + } + + /* and finally shrink the tree if our parent is the root with 1 */ + if (le16_to_cpu(parent->nr_items) == 1) { + root->height--; + root->ref.blkno = bt->hdr.blkno; + root->ref.seq = bt->hdr.seq; + ret = scoutfs_free_meta(sb, alloc, wri, + le64_to_cpu(parent->hdr.blkno)); + BUG_ON(ret); + } + + scoutfs_block_put(sb, sib_bl); + + return 1; +} + +static bool bad_item_off(int off, int nr) +{ + return (off < offsetof(struct scoutfs_btree_block, items[0])) || + (off >= offsetof(struct scoutfs_btree_block, items[nr])) || + ((off - offsetof(struct scoutfs_btree_block, items[0])) + % sizeof(struct scoutfs_btree_item)); +} + +static bool bad_avl_node_off(__le16 node_off, int nr) +{ + int item_off; + + if (node_off == 0) + return false; + + item_off = (int)le16_to_cpu(node_off) + + offsetof(struct scoutfs_btree_block, item_root) - + offsetof(struct scoutfs_btree_item, node); + + return bad_item_off(item_off, nr); +} + +/* + * XXX: + * - values don't overlap items + * - values don't overlap each other + * - last_free_offset is in fact last free region + * - call after leaf modification + * - padding is zero + */ +static void verify_btree_block(struct super_block *sb, + struct scoutfs_btree_block *bt, int level, + struct scoutfs_key *start, + struct scoutfs_key *end) +{ + __le16 *buckets = leaf_item_hash_buckets(bt); + struct scoutfs_btree_item *item; + char *reason = NULL; + int first_val = 0; + int hashed = 0; + int end_off; + int tot = 0; + int i = 0; + int j = 0; + int nr; + + if (bt->level != level) { + reason = "unexpected level"; + goto out; + } + + BUILD_BUG_ON(SCOUTFS_BTREE_LEAF_ITEM_HASH_BYTES % SCOUTFS_BTREE_VALUE_ALIGN != 0); + + end_off = SCOUTFS_BLOCK_LG_SIZE - + (level ? 0 : SCOUTFS_BTREE_LEAF_ITEM_HASH_BYTES); + + /* can have 0 item blocks during first insertion into a tree */ + nr = le16_to_cpu(bt->nr_items); + if (nr < 0 || nr > SCOUTFS_BLOCK_LG_SIZE || + offsetof(struct scoutfs_btree_block, items[nr]) > end_off) { + reason = "nr_items out of range"; + goto out; + } + + if (bad_avl_node_off(bt->item_root.node, nr)) { + reason = "item_root node off"; + goto out; + } + + tot = 0; + first_val = end_off; + + for (i = 0; i < le16_to_cpu(bt->nr_items); i++) { + item = &bt->items[i]; + + if (bad_avl_node_off(item->node.parent, nr) || + bad_avl_node_off(item->node.left, nr) || + bad_avl_node_off(item->node.right, nr)) { + reason = "item node off"; + goto out; + } + + for (j = 0; j < sizeof(item->__pad); j++) { + WARN_ON_ONCE(item->__pad[j] != 0); + } + + if (scoutfs_key_compare(&item->key, start) < 0 || + scoutfs_key_compare(&item->key, end) > 0) { + reason = "item key out of parent range"; + goto out; + } + + if (level == 0 && + leaf_item_hash_search(sb, bt, &item->key) != item) { + reason = "item not found in hash"; + goto out; + } + + if (le16_to_cpu(item->val_len) > SCOUTFS_BTREE_MAX_VAL_LEN) { + reason = "bad item val len"; + goto out; + } + + if (((int)le16_to_cpu(item->val_off) + + le16_to_cpu(item->val_len)) > end_off) { + reason = "item value outside valid"; + goto out; + } + + tot += sizeof(struct scoutfs_btree_item) + + le16_to_cpu(item->val_len); + + if (item->val_len != 0) { + first_val = min_t(int, first_val, + le16_to_cpu(item->val_off)); + } + } + + for (i = 0; level == 0 && i < SCOUTFS_BTREE_LEAF_ITEM_HASH_NR; i++) { + if (buckets[i] == 0) + continue; + + if (bad_item_off(le16_to_cpu(buckets[i]), nr)) { + reason = "bad item hash offset"; + goto out; + } + + hashed++; + } + + if (level == 0 && hashed != nr) { + reason = "set hash buckets not nr"; + goto out; + } + + if (le16_to_cpu(bt->total_item_bytes) != tot) { + reason = "total_item_bytes not sum of items"; + goto out; + } + + /* value deletion doesn't merge with adjacent fragmented freed vals */ + if (le16_to_cpu(bt->mid_free_len) > + (first_val - offsetof(struct scoutfs_btree_block, items[nr]))) { + reason = "mid_free_len too large"; + goto out; + } +out: + if (!reason) + return; + + printk("found btree block inconsistency: %s\n", reason); + printk("start "SK_FMT" end "SK_FMT"\n", SK_ARG(start), SK_ARG(end)); + printk("calced: i %u tot %u hashed %u fv %u\n", + i, tot, hashed, first_val); + + printk("hdr: crc %x magic %x fsid %llx seq %llx blkno %llu\n", + le32_to_cpu(bt->hdr.crc), le32_to_cpu(bt->hdr.magic), + le64_to_cpu(bt->hdr.fsid), le64_to_cpu(bt->hdr.seq), + le64_to_cpu(bt->hdr.blkno)); + printk("item_root: node %u\n", le16_to_cpu(bt->item_root.node)); + printk("nr %u tib %u mfl %u lvl %u\n", + le16_to_cpu(bt->nr_items), le16_to_cpu(bt->total_item_bytes), + le16_to_cpu(bt->mid_free_len), bt->level); + + for (i = 0; i < le16_to_cpu(bt->nr_items); i++) { + item = &bt->items[i]; + printk(" %u: n %u,%u,%u,%u k "SK_FMT" vo %u vl %u\n", + i, le16_to_cpu(item->node.parent), + le16_to_cpu(item->node.left), + le16_to_cpu(item->node.right), item->node.height, + SK_ARG(&item->key), le16_to_cpu(item->val_off), + le16_to_cpu(item->val_len)); + } + + BUG(); +} + +struct btree_walk_key_range { + struct scoutfs_key start; + struct scoutfs_key end; + /* zero if no remaining blocks outside our walk in that direction */ + struct scoutfs_key iter_prev; + struct scoutfs_key iter_next; +}; + +/* + * Return the leaf block that should contain the given key. The caller + * is responsible for searching the leaf block and performing their + * operation. + * + * Iteration starting from a key can end up in a leaf that doesn't + * contain the next item in the direction iteration. As we descend we + * give the caller the nearest key in the direction of iteration that + * will land in a different leaf. + * + * Migrating is a special kind of dirtying that returns the parent block + * in the walk if the leaf block is already current and doesn't need to + * be migrated. It's presumed that the caller is iterating over keys + * dirtying old leaf blocks and isn't actually doing anything with the + * blocks themselves. + */ +static int btree_walk(struct super_block *sb, + struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_btree_root *root, + int flags, struct scoutfs_key *key, + unsigned int val_len, + struct scoutfs_block **bl_ret, + struct btree_walk_key_range *kr) +{ + struct scoutfs_block *par_bl = NULL; + struct scoutfs_block *bl = NULL; + struct scoutfs_btree_block *parent = NULL; + struct scoutfs_btree_block *bt; + struct scoutfs_btree_item *par_item; + struct scoutfs_btree_item *item; + struct scoutfs_btree_item *prev; + struct scoutfs_avl_node *next_node; + struct scoutfs_avl_node *node; + struct scoutfs_btree_ref *ref; + unsigned int level; + unsigned int nr; + int ret; + + if (WARN_ON_ONCE((flags & BTW_DIRTY) && (!alloc || !wri))) + return -EINVAL; + + scoutfs_inc_counter(sb, btree_walk); + +restart: + scoutfs_block_put(sb, par_bl); + par_bl = NULL; + parent = NULL; + par_item = NULL; + scoutfs_block_put(sb, bl); + bl = NULL; + bt = NULL; + if (kr) { + scoutfs_key_set_zeros(&kr->start); + scoutfs_key_set_ones(&kr->end); + scoutfs_key_set_zeros(&kr->iter_prev); + scoutfs_key_set_zeros(&kr->iter_next); + } + level = root->height; + ret = 0; + + if (!root->height) { + if (!(flags & BTW_INSERT)) { + ret = -ENOENT; + } else { + ret = get_ref_block(sb, alloc, wri, BTW_ALLOC, + &root->ref, &bl); + if (ret == 0) { + bt = bl->data; + init_btree_block(bt, 0); + root->height = 1; + } + } + goto out; + } + + ref = &root->ref; + + while(level-- > 0) { + + trace_scoutfs_btree_walk(sb, root, key, flags, level, ref); + + ret = get_ref_block(sb, alloc, wri, flags, ref, &bl); + if (ret) + break; + bt = bl->data; + + if (0 && kr) + verify_btree_block(sb, bt, level, &kr->start, &kr->end); + + /* XXX more aggressive block verification, before ref updates? */ + if (bt->level != level) { + scoutfs_corruption(sb, SC_BTREE_BLOCK_LEVEL, + corrupt_btree_block_level, + "root_height %u root_blkno %llu root_seq %llu blkno %llu seq %llu level %u expected %u", + root->height, + le64_to_cpu(root->ref.blkno), + le64_to_cpu(root->ref.seq), + le64_to_cpu(bt->hdr.blkno), + le64_to_cpu(bt->hdr.seq), bt->level, + level); + ret = -EIO; + break; + } + + /* + * Splitting and joining can add or remove parents or + * change the parent item we use to reach the child + * block with the search key. In the rare case that we + * split or join we simply restart the walk instead of + * update our state to reflect the tree changes. + */ + ret = 0; + if (flags & (BTW_INSERT | BTW_DELETE)) + ret = try_split(sb, alloc, wri, root, key, val_len, + parent, bt); + if (ret == 0 && (flags & BTW_DELETE) && parent) + ret = try_join(sb, alloc, wri, root, parent, par_item, + bt); + if (ret > 0) { + scoutfs_inc_counter(sb, btree_walk_restart); + goto restart; + } + else if (ret < 0) + break; + + /* done at the leaf */ + if (level == 0) + break; + + nr = le16_to_cpu(bt->nr_items); + /* Find the next child block for the search key. */ + node = scoutfs_avl_search(&bt->item_root, cmp_key_item, key, + NULL, NULL, &next_node, NULL); + item = node_item(node ?: next_node); + if (item == NULL) { + scoutfs_corruption(sb, SC_BTREE_NO_CHILD_REF, + corrupt_btree_block_level, + "root_height %u root_blkno %llu root_seq %llu blkno %llu seq %llu level %u nr %u", + root->height, + le64_to_cpu(root->ref.blkno), + le64_to_cpu(root->ref.seq), + le64_to_cpu(bt->hdr.blkno), + le64_to_cpu(bt->hdr.seq), bt->level, + nr); + ret = -EIO; + break; + } + + if (kr) { + /* update keys for walk bounds and next iteration */ + if ((prev = prev_item(bt, item))) { + kr->start = *item_key(prev); + scoutfs_key_inc(&kr->start); + kr->iter_prev = *item_key(prev); + } + kr->end = *item_key(item); + if (next_item(bt, item)) { + kr->iter_next = *item_key(item); + scoutfs_key_inc(&kr->iter_next); + } + } + + scoutfs_block_put(sb, par_bl); + par_bl = bl; + parent = bt; + bl = NULL; + bt = NULL; + + par_item = item; + ref = item_val(parent, par_item); + } + +out: + scoutfs_block_put(sb, par_bl); + if (ret) { + scoutfs_block_put(sb, bl); + bl = NULL; + } + + if (bl_ret) + *bl_ret = bl; + else + scoutfs_block_put(sb, bl); + + return ret; +} + +static void init_item_ref(struct scoutfs_btree_item_ref *iref, + struct super_block *sb, + struct scoutfs_block *bl, + struct scoutfs_btree_item *item) +{ + struct scoutfs_btree_block *bt = bl->data; + + iref->sb = sb; + iref->bl = bl; + iref->key = item_key(item); + iref->val = item_val(bt, item); + iref->val_len = le16_to_cpu(item->val_len); +} + +void scoutfs_btree_put_iref(struct scoutfs_btree_item_ref *iref) +{ + if (!IS_ERR_OR_NULL(iref) && !IS_ERR_OR_NULL(iref->bl)) { + scoutfs_block_put(iref->sb, iref->bl); + memset(iref, 0, sizeof(struct scoutfs_btree_item_ref)); + } +} + +/* + * Find the item with the given key and point to it from the caller's + * item ref. They're given a reference to the block that they'll drop + * when they're done. + */ +int scoutfs_btree_lookup(struct super_block *sb, + struct scoutfs_btree_root *root, + struct scoutfs_key *key, + struct scoutfs_btree_item_ref *iref) +{ + struct scoutfs_btree_item *item; + struct scoutfs_btree_block *bt; + struct scoutfs_block *bl; + int ret; + + scoutfs_inc_counter(sb, btree_lookup); + + if (WARN_ON_ONCE(iref->key)) + return -EINVAL; + + ret = btree_walk(sb, NULL, NULL, root, 0, key, 0, &bl, NULL); + if (ret == 0) { + bt = bl->data; + + item = leaf_item_hash_search(sb, bt, key); + if (item) { + init_item_ref(iref, sb, bl, item); + ret = 0; + } else { + scoutfs_block_put(sb, bl); + ret = -ENOENT; + } + } + + return ret; +} + +static bool invalid_item(unsigned val_len) +{ + return WARN_ON_ONCE(val_len > SCOUTFS_BTREE_MAX_VAL_LEN); +} + +/* + * Insert a new item in the tree. + * + * 0 is returned on success. -EEXIST is returned if the key is already + * present in the tree. + * + * If no value pointer is given then the item is created with a zero + * length value. + */ +int scoutfs_btree_insert(struct super_block *sb, + struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_btree_root *root, + struct scoutfs_key *key, + void *val, unsigned val_len) +{ + struct scoutfs_btree_item *item; + struct scoutfs_btree_block *bt; + struct scoutfs_avl_node *node; + struct scoutfs_avl_node *par; + struct scoutfs_block *bl; + int cmp; + int ret; + + scoutfs_inc_counter(sb, btree_insert); + + if (invalid_item(val_len)) + return -EINVAL; + + ret = btree_walk(sb, alloc, wri, root, BTW_DIRTY | BTW_INSERT, key, + val_len, &bl, NULL); + if (ret == 0) { + bt = bl->data; + + item = leaf_item_hash_search(sb, bt, key); + if (item) { + ret = -EEXIST; + } else { + node = scoutfs_avl_search(&bt->item_root, cmp_key_item, + key, &cmp, &par, NULL, NULL); + if (node) { + ret = -EEXIST; + } else { + create_item(bt, key, val, val_len, par, cmp); + ret = 0; + } + } + + scoutfs_block_put(sb, bl); + } + + return ret; +} + +static void update_item_value(struct scoutfs_btree_block *bt, + struct scoutfs_btree_item *item, + void *val, unsigned val_len) +{ + le16_add_cpu(&bt->total_item_bytes, val_bytes(val_len) - + val_bytes(le16_to_cpu(item->val_len))); + item->val_off = insert_value(bt, ptr_off(bt, item), val, val_len); + item->val_len = cpu_to_le16(val_len); +} + +/* + * Update a btree item. -ENOENT is returned if the item didn't exist. + * + * We don't know the existing item's value length as we first descend. + * We assume that the new value is longer and try to split so that we + * can insert if that's true. If the new value is shorter than the + * existing then the leaf might fall under the minimum watermark, but at + * least we can do that while we simply can't insert a new longer value + * which doesn't fit. + */ +int scoutfs_btree_update(struct super_block *sb, + struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_btree_root *root, + struct scoutfs_key *key, + void *val, unsigned val_len) +{ + struct scoutfs_btree_item *item; + struct scoutfs_btree_block *bt; + struct scoutfs_block *bl; + int ret; + + scoutfs_inc_counter(sb, btree_update); + + if (invalid_item(val_len)) + return -EINVAL; + + ret = btree_walk(sb, alloc, wri, root, BTW_DIRTY | BTW_INSERT, key, + val_len, &bl, NULL); + if (ret == 0) { + bt = bl->data; + + item = leaf_item_hash_search(sb, bt, key); + if (item) { + update_item_value(bt, item, val, val_len); + ret = 0; + } else { + ret = -ENOENT; + } + + scoutfs_block_put(sb, bl); + } + + return ret; +} + +/* + * Create an item, overwriting any item that might exist. It's _update + * which will insert instead of returning -ENOENT. + */ +int scoutfs_btree_force(struct super_block *sb, + struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_btree_root *root, + struct scoutfs_key *key, + void *val, unsigned val_len) +{ + struct scoutfs_btree_item *item; + struct scoutfs_avl_node *par; + struct scoutfs_btree_block *bt; + struct scoutfs_block *bl; + int cmp; + int ret; + + scoutfs_inc_counter(sb, btree_force); + + if (invalid_item(val_len)) + return -EINVAL; + + ret = btree_walk(sb, alloc, wri, root, BTW_DIRTY | BTW_INSERT, key, + val_len, &bl, NULL); + if (ret == 0) { + bt = bl->data; + + item = leaf_item_hash_search(sb, bt, key); + if (item) { + update_item_value(bt, item, val, val_len); + } else { + scoutfs_avl_search(&bt->item_root, cmp_key_item, key, + &cmp, &par, NULL, NULL); + create_item(bt, key, val, val_len, par, cmp); + } + ret = 0; + + scoutfs_block_put(sb, bl); + } + + return ret; +} + +/* + * Delete an item from the tree. -ENOENT is returned if the key isn't + * found. + */ +int scoutfs_btree_delete(struct super_block *sb, + struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_btree_root *root, + struct scoutfs_key *key) +{ + struct scoutfs_btree_item *item; + struct scoutfs_btree_block *bt; + struct scoutfs_block *bl; + int ret; + + scoutfs_inc_counter(sb, btree_delete); + + ret = btree_walk(sb, alloc, wri, root, BTW_DELETE | BTW_DIRTY, key, + 0, &bl, NULL); + if (ret == 0) { + bt = bl->data; + + item = leaf_item_hash_search(sb, bt, key); + if (item) { + if (le16_to_cpu(bt->nr_items) == 1) { + /* remove final empty block */ + ret = scoutfs_free_meta(sb, alloc, wri, + bl->blkno); + if (ret == 0) { + root->height = 0; + root->ref.blkno = 0; + root->ref.seq = 0; + } + } else { + delete_item(bt, item, NULL); + ret = 0; + } + } else { + ret = -ENOENT; + } + + scoutfs_block_put(sb, bl); + } + + return ret; +} + +/* + * Iterate from a key value to the next item in the direction of + * iteration. Callers set flags to tell which way to iterate. The + * first key is always inclusive. + * + * Walking can land in a leaf that doesn't contain any items in the + * direction of the iteration. Walking gives us the next key to walk + * towards in this case. We keep trying until we run out of blocks or + * find the next item. This method is aggressively permissive because + * it lets the tree shape change between each walk and allows empty + * blocks. + */ +static int btree_iter(struct super_block *sb,struct scoutfs_btree_root *root, + int flags, struct scoutfs_key *key, + struct scoutfs_btree_item_ref *iref) +{ + struct scoutfs_avl_node *node; + struct scoutfs_avl_node *next; + struct scoutfs_avl_node *prev; + struct scoutfs_btree_item *item; + struct scoutfs_btree_block *bt; + struct btree_walk_key_range kr; + struct scoutfs_key walk_key; + struct scoutfs_key *iter_key; + struct scoutfs_block *bl; + int ret; + + if (WARN_ON_ONCE(flags & BTW_DIRTY) || + WARN_ON_ONCE(iref->key)) + return -EINVAL; + + walk_key = *key; + + for (;;) { + ret = btree_walk(sb, NULL, NULL, root, flags, &walk_key, + 0, &bl, &kr); + if (ret < 0) + break; + bt = bl->data; + + node = scoutfs_avl_search(&bt->item_root, cmp_key_item, key, + NULL, NULL, &next, &prev); + + if (node == NULL && (flags & BTW_NEXT)) + node = next; + else if (node == NULL && (flags & BTW_PREV)) + node = prev; + item = node_item(node); + if (item) { + init_item_ref(iref, sb, bl, item); + ret = 0; + break; + } + + scoutfs_block_put(sb, bl); + + /* nothing in this leaf, walk gave us a key */ + iter_key = (flags & BTW_NEXT) ? &kr.iter_next : &kr.iter_prev; + if (!scoutfs_key_is_zeros(iter_key)) { + walk_key = *iter_key; + continue; + } + + ret = -ENOENT; + break; + } + + return ret; +} + +int scoutfs_btree_next(struct super_block *sb, struct scoutfs_btree_root *root, + struct scoutfs_key *key, + struct scoutfs_btree_item_ref *iref) +{ + scoutfs_inc_counter(sb, btree_next); + + return btree_iter(sb, root, BTW_NEXT, key, iref); +} + +int scoutfs_btree_prev(struct super_block *sb, struct scoutfs_btree_root *root, + struct scoutfs_key *key, + struct scoutfs_btree_item_ref *iref) +{ + scoutfs_inc_counter(sb, btree_prev); + + return btree_iter(sb, root, BTW_PREV, key, iref); +} + +/* + * Ensure that the blocks that lead to the item with the given key are + * dirty. caller can hold a transaction to pin the dirty blocks and + * guarantee that later updates of the item will succeed. + * + * <0 is returned on error, including -ENOENT if the key isn't present. + */ +int scoutfs_btree_dirty(struct super_block *sb, + struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_btree_root *root, + struct scoutfs_key *key) +{ + struct scoutfs_btree_item *item; + struct scoutfs_btree_block *bt; + struct scoutfs_block *bl; + int ret; + + scoutfs_inc_counter(sb, btree_dirty); + + ret = btree_walk(sb, alloc, wri, root, BTW_DIRTY, key, 0, &bl, NULL); + if (ret == 0) { + bt = bl->data; + + item = leaf_item_hash_search(sb, bt, key); + if (item) + ret = 0; + else + ret = -ENOENT; + + scoutfs_block_put(sb, bl); + } + + return ret; +} + +/* + * Call the users callback on all the items in the leaf that we find. + * We also set the caller's keys for the first and last possible keys + * that could exist in the leaf block. + */ +int scoutfs_btree_read_items(struct super_block *sb, + struct scoutfs_btree_root *root, + struct scoutfs_key *key, + struct scoutfs_key *start, + struct scoutfs_key *end, + scoutfs_btree_item_cb cb, void *arg) +{ + struct scoutfs_btree_item *item; + struct scoutfs_btree_block *bt; + struct scoutfs_avl_node *next_node; + struct scoutfs_avl_node *node; + struct btree_walk_key_range kr; + struct scoutfs_block *bl; + int ret; + + ret = btree_walk(sb, NULL, NULL, root, 0, key, 0, &bl, &kr); + if (ret < 0) + goto out; + bt = bl->data; + + if (scoutfs_key_compare(&kr.start, start) > 0) + *start = kr.start; + if (scoutfs_key_compare(&kr.end, end) < 0) + *end = kr.end; + + node = scoutfs_avl_search(&bt->item_root, cmp_key_item, start, NULL, + NULL, &next_node, NULL) ?: next_node; + while (node) { + item = node_item(node); + if (scoutfs_key_compare(&item->key, end) > 0) + break; + + ret = cb(sb, item_key(item), item_val(bt, item), + item_val_len(item), arg); + if (ret < 0) + break; + + node = scoutfs_avl_next(&bt->item_root, node); + } + + scoutfs_block_put(sb, bl); +out: + return ret; +} + +/* + * The caller has a sorted list of items to insert. We find the leaf + * block that contains each item and either overwrite or insert the + * caller's item. This has no mechanism for deleting items. + * + * This can make partial progress before returning an error, leaving + * dirty btree blocks with only some of the caller's items. It's up to + * the caller to resolve this. + */ +int scoutfs_btree_insert_list(struct super_block *sb, + struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_btree_root *root, + struct scoutfs_btree_item_list *lst) +{ + struct scoutfs_btree_item *item; + struct btree_walk_key_range kr; + struct scoutfs_btree_block *bt; + struct scoutfs_avl_node *par; + struct scoutfs_block *bl; + int cmp; + int ret = 0; + + while (lst) { + ret = btree_walk(sb, alloc, wri, root, BTW_DIRTY | BTW_INSERT, + &lst->key, lst->val_len, &bl, &kr); + if (ret < 0) + goto out; + bt = bl->data; + + do { + item = leaf_item_hash_search(sb, bt, &lst->key); + if (item) { + update_item_value(bt, item, lst->val, + lst->val_len); + } else { + scoutfs_avl_search(&bt->item_root, + cmp_key_item, &lst->key, + &cmp, &par, NULL, NULL); + create_item(bt, &lst->key, lst->val, + lst->val_len, par, cmp); + } + + lst = lst->next; + } while (lst && scoutfs_key_compare(&lst->key, &kr.end) <= 0 && + mid_free_item_room(bt, lst->val_len)); + + scoutfs_block_put(sb, bl); + } + +out: + return ret; +} diff --git a/kmod/src/btree.h b/kmod/src/btree.h new file mode 100644 index 00000000..79d4de58 --- /dev/null +++ b/kmod/src/btree.h @@ -0,0 +1,87 @@ +#ifndef _SCOUTFS_BTREE_H_ +#define _SCOUTFS_BTREE_H_ + +#include + +struct scoutfs_alloc; +struct scoutfs_block_writer; +struct scoutfs_block; + +struct scoutfs_btree_item_ref { + struct super_block *sb; + struct scoutfs_block *bl; + struct scoutfs_key *key; + void *val; + unsigned val_len; +}; + +#define SCOUTFS_BTREE_ITEM_REF(name) \ + struct scoutfs_btree_item_ref name = {NULL,} + +/* caller gives an item to the callback */ +typedef int (*scoutfs_btree_item_cb)(struct super_block *sb, + struct scoutfs_key *key, + void *val, int val_len, void *arg); + +/* simple singly-linked list of items */ +struct scoutfs_btree_item_list { + struct scoutfs_btree_item_list *next; + struct scoutfs_key key; + int val_len; + u8 val[0]; +}; + +int scoutfs_btree_lookup(struct super_block *sb, + struct scoutfs_btree_root *root, + struct scoutfs_key *key, + struct scoutfs_btree_item_ref *iref); +int scoutfs_btree_insert(struct super_block *sb, + struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_btree_root *root, + struct scoutfs_key *key, + void *val, unsigned val_len); +int scoutfs_btree_update(struct super_block *sb, + struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_btree_root *root, + struct scoutfs_key *key, + void *val, unsigned val_len); +int scoutfs_btree_force(struct super_block *sb, + struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_btree_root *root, + struct scoutfs_key *key, + void *val, unsigned val_len); +int scoutfs_btree_delete(struct super_block *sb, + struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_btree_root *root, + struct scoutfs_key *key); +int scoutfs_btree_next(struct super_block *sb, struct scoutfs_btree_root *root, + struct scoutfs_key *key, + struct scoutfs_btree_item_ref *iref); +int scoutfs_btree_prev(struct super_block *sb, struct scoutfs_btree_root *root, + struct scoutfs_key *key, + struct scoutfs_btree_item_ref *iref); +int scoutfs_btree_dirty(struct super_block *sb, + struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_btree_root *root, + struct scoutfs_key *key); + +int scoutfs_btree_read_items(struct super_block *sb, + struct scoutfs_btree_root *root, + struct scoutfs_key *key, + struct scoutfs_key *start, + struct scoutfs_key *end, + scoutfs_btree_item_cb cb, void *arg); +int scoutfs_btree_insert_list(struct super_block *sb, + struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_btree_root *root, + struct scoutfs_btree_item_list *lst); + +void scoutfs_btree_put_iref(struct scoutfs_btree_item_ref *iref); + +#endif diff --git a/kmod/src/client.c b/kmod/src/client.c new file mode 100644 index 00000000..305794ca --- /dev/null +++ b/kmod/src/client.c @@ -0,0 +1,564 @@ +/* + * Copyright (C) 2017 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "format.h" +#include "counters.h" +#include "inode.h" +#include "btree.h" +#include "scoutfs_trace.h" +#include "msg.h" +#include "client.h" +#include "net.h" +#include "endian_swap.h" +#include "quorum.h" + +/* + * The client is responsible for maintaining a connection to the server. + * This includes managing quorum elections that determine which client + * should run the server that all the clients connect to. + */ + +#define CLIENT_CONNECT_DELAY_MS (MSEC_PER_SEC / 10) +#define CLIENT_CONNECT_TIMEOUT_MS (1 * MSEC_PER_SEC) +#define CLIENT_QUORUM_TIMEOUT_MS (5 * MSEC_PER_SEC) + +struct client_info { + struct super_block *sb; + + struct scoutfs_net_connection *conn; + atomic_t shutting_down; + + struct workqueue_struct *workq; + struct delayed_work connect_dwork; + + u64 server_term; + u64 greeting_umb; + + bool sending_farewell; + int farewell_error; + struct completion farewell_comp; +}; + +/* + * Ask for a new run of allocated inode numbers. The server can return + * fewer than @count. It will success with nr == 0 if we've run out. + */ +int scoutfs_client_alloc_inodes(struct super_block *sb, u64 count, + u64 *ino, u64 *nr) +{ + struct client_info *client = SCOUTFS_SB(sb)->client_info; + struct scoutfs_net_inode_alloc ial; + __le64 lecount = cpu_to_le64(count); + int ret; + + ret = scoutfs_net_sync_request(sb, client->conn, + SCOUTFS_NET_CMD_ALLOC_INODES, + &lecount, sizeof(lecount), + &ial, sizeof(ial)); + if (ret == 0) { + *ino = le64_to_cpu(ial.ino); + *nr = le64_to_cpu(ial.nr); + + if (*nr == 0) + ret = -ENOSPC; + else if (*ino + *nr < *ino) + ret = -EINVAL; + } + + return ret; +} + +int scoutfs_client_get_log_trees(struct super_block *sb, + struct scoutfs_log_trees *lt) +{ + struct client_info *client = SCOUTFS_SB(sb)->client_info; + + return scoutfs_net_sync_request(sb, client->conn, + SCOUTFS_NET_CMD_GET_LOG_TREES, + NULL, 0, lt, sizeof(*lt)); +} + +int scoutfs_client_commit_log_trees(struct super_block *sb, + struct scoutfs_log_trees *lt) +{ + struct client_info *client = SCOUTFS_SB(sb)->client_info; + + return scoutfs_net_sync_request(sb, client->conn, + SCOUTFS_NET_CMD_COMMIT_LOG_TREES, + lt, sizeof(*lt), NULL, 0); +} + +int scoutfs_client_get_roots(struct super_block *sb, + struct scoutfs_net_roots *roots) +{ + struct client_info *client = SCOUTFS_SB(sb)->client_info; + + return scoutfs_net_sync_request(sb, client->conn, + SCOUTFS_NET_CMD_GET_ROOTS, + NULL, 0, roots, sizeof(*roots)); +} + +int scoutfs_client_advance_seq(struct super_block *sb, u64 *seq) +{ + struct client_info *client = SCOUTFS_SB(sb)->client_info; + __le64 before = cpu_to_le64p(seq); + __le64 after; + int ret; + + ret = scoutfs_net_sync_request(sb, client->conn, + SCOUTFS_NET_CMD_ADVANCE_SEQ, + &before, sizeof(before), + &after, sizeof(after)); + if (ret == 0) + *seq = le64_to_cpu(after); + + return ret; +} + +int scoutfs_client_get_last_seq(struct super_block *sb, u64 *seq) +{ + struct client_info *client = SCOUTFS_SB(sb)->client_info; + __le64 last_seq; + int ret; + + ret = scoutfs_net_sync_request(sb, client->conn, + SCOUTFS_NET_CMD_GET_LAST_SEQ, + NULL, 0, &last_seq, sizeof(last_seq)); + if (ret == 0) + *seq = le64_to_cpu(last_seq); + + return ret; +} + +/* process an incoming grant response from the server */ +static int client_lock_response(struct super_block *sb, + struct scoutfs_net_connection *conn, + void *resp, unsigned int resp_len, + int error, void *data) +{ + if (resp_len != sizeof(struct scoutfs_net_lock_grant_response)) + return -EINVAL; + + /* XXX error? */ + + return scoutfs_lock_grant_response(sb, resp); +} + +/* Send a lock request to the server. */ +int scoutfs_client_lock_request(struct super_block *sb, + struct scoutfs_net_lock *nl) +{ + struct client_info *client = SCOUTFS_SB(sb)->client_info; + + return scoutfs_net_submit_request(sb, client->conn, + SCOUTFS_NET_CMD_LOCK, + nl, sizeof(*nl), + client_lock_response, NULL, NULL); +} + +/* Send a lock response to the server. */ +int scoutfs_client_lock_response(struct super_block *sb, u64 net_id, + struct scoutfs_net_lock *nl) +{ + struct client_info *client = SCOUTFS_SB(sb)->client_info; + + return scoutfs_net_response(sb, client->conn, SCOUTFS_NET_CMD_LOCK, + net_id, 0, nl, sizeof(*nl)); +} + +/* Send a lock recover response to the server. */ +int scoutfs_client_lock_recover_response(struct super_block *sb, u64 net_id, + struct scoutfs_net_lock_recover *nlr) +{ + struct client_info *client = SCOUTFS_SB(sb)->client_info; + u16 bytes = offsetof(struct scoutfs_net_lock_recover, + locks[le16_to_cpu(nlr->nr)]); + + return scoutfs_net_response(sb, client->conn, + SCOUTFS_NET_CMD_LOCK_RECOVER, + net_id, 0, nlr, bytes); +} + +/* Find srch files that need to be compacted. */ +int scoutfs_client_srch_get_compact(struct super_block *sb, + struct scoutfs_srch_compact *sc) +{ + struct client_info *client = SCOUTFS_SB(sb)->client_info; + + return scoutfs_net_sync_request(sb, client->conn, + SCOUTFS_NET_CMD_SRCH_GET_COMPACT, + NULL, 0, sc, sizeof(*sc)); +} + +/* Commit the result of a srch file compaction. */ +int scoutfs_client_srch_commit_compact(struct super_block *sb, + struct scoutfs_srch_compact *res) +{ + struct client_info *client = SCOUTFS_SB(sb)->client_info; + + return scoutfs_net_sync_request(sb, client->conn, + SCOUTFS_NET_CMD_SRCH_COMMIT_COMPACT, + res, sizeof(*res), NULL, 0); +} + +/* The client is receiving a invalidation request from the server */ +static int client_lock(struct super_block *sb, + struct scoutfs_net_connection *conn, u8 cmd, u64 id, + void *arg, u16 arg_len) +{ + if (arg_len != sizeof(struct scoutfs_net_lock)) + return -EINVAL; + + /* XXX error? */ + + return scoutfs_lock_invalidate_request(sb, id, arg); +} + +/* The server is asking us for the client's locks starting with the given key */ +static int client_lock_recover(struct super_block *sb, + struct scoutfs_net_connection *conn, + u8 cmd, u64 id, void *arg, u16 arg_len) +{ + if (arg_len != sizeof(struct scoutfs_key)) + return -EINVAL; + + /* XXX error? */ + + return scoutfs_lock_recover_request(sb, id, arg); +} + +/* + * Process a greeting response in the client from the server. This is + * called for every connected socket on the connection. Each response + * contains the remote server's elected term which can be used to + * identify server failover. + */ +static int client_greeting(struct super_block *sb, + struct scoutfs_net_connection *conn, + void *resp, unsigned int resp_len, int error, + void *data) +{ + struct client_info *client = SCOUTFS_SB(sb)->client_info; + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + struct scoutfs_net_greeting *gr = resp; + bool new_server; + int ret; + + if (error) { + ret = error; + goto out; + } + + if (resp_len != sizeof(struct scoutfs_net_greeting)) { + ret = -EINVAL; + goto out; + } + + if (gr->fsid != super->hdr.fsid) { + scoutfs_warn(sb, "server sent fsid 0x%llx, client has 0x%llx", + le64_to_cpu(gr->fsid), + le64_to_cpu(super->hdr.fsid)); + ret = -EINVAL; + goto out; + } + + if (gr->format_hash != super->format_hash) { + scoutfs_warn(sb, "server sent format 0x%llx, client has 0x%llx", + le64_to_cpu(gr->format_hash), + le64_to_cpu(super->format_hash)); + ret = -EINVAL; + goto out; + } + + new_server = le64_to_cpu(gr->server_term) != client->server_term; + scoutfs_net_client_greeting(sb, conn, new_server); + + client->server_term = le64_to_cpu(gr->server_term); + client->greeting_umb = le64_to_cpu(gr->unmount_barrier); + ret = 0; +out: + return ret; +} + +/* + * This work is responsible for maintaining a connection from the client + * to the server. It's queued on mount and disconnect and we requeue + * the work if the work fails and we're not shutting down. + * + * In the typical case a mount reads the super blocks and finds the + * address of the currently running server and connects to it. + * Non-voting clients who can't connect will keep trying alternating + * reading the address and getting connect timeouts. + * + * Voting mounts will try to elect a leader if they can't connect to the + * server. When a quorum can't connect and are able to elect a leader + * then a new server is started. The new server will write its address + * in the super and everyone will be able to connect. + * + * There's a tricky bit of coordination required to safely unmount. + * Clients need to tell the server that they won't be coming back with a + * farewell request. Once a client receives its farewell response it + * can exit. But a majority of clients need to stick around to elect a + * server to process all their farewell requests. This is coordinated + * by having the greeting tell the server that a client is a voter. The + * server then holds on to farewell requests from voters until only + * requests from the final quorum remain. These farewell responses are + * only sent after updating an unmount barrier in the super to indicate + * to the final quorum that they can safely exit without having received + * a farewell response over the network. + */ +static void scoutfs_client_connect_worker(struct work_struct *work) +{ + struct client_info *client = container_of(work, struct client_info, + connect_dwork.work); + struct super_block *sb = client->sb; + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = NULL; + struct mount_options *opts = &sbi->opts; + const bool am_voter = opts->server_addr.sin_addr.s_addr != 0; + struct scoutfs_net_greeting greet; + struct sockaddr_in sin; + ktime_t timeout_abs; + u64 elected_term; + int ret; + + super = kmalloc(sizeof(struct scoutfs_super_block), GFP_NOFS); + if (!super) { + ret = -ENOMEM; + goto out; + } + + ret = scoutfs_read_super(sb, super); + if (ret) + goto out; + + /* can safely unmount if we see that server processed our farewell */ + if (am_voter && client->sending_farewell && + (le64_to_cpu(super->unmount_barrier) > client->greeting_umb)) { + client->farewell_error = 0; + complete(&client->farewell_comp); + ret = 0; + goto out; + } + + /* try to connect to the super's server address */ + scoutfs_addr_to_sin(&sin, &super->server_addr); + if (sin.sin_addr.s_addr != 0 && sin.sin_port != 0) + ret = scoutfs_net_connect(sb, client->conn, &sin, + CLIENT_CONNECT_TIMEOUT_MS); + else + ret = -ENOTCONN; + + /* voters try to elect a leader if they couldn't connect */ + if (ret < 0) { + /* non-voters will keep retrying */ + if (!am_voter) + goto out; + + /* make sure local server isn't writing super during votes */ + scoutfs_server_stop(sb); + + timeout_abs = ktime_add_ms(ktime_get(), + CLIENT_QUORUM_TIMEOUT_MS); + + ret = scoutfs_quorum_election(sb, timeout_abs, + le64_to_cpu(super->quorum_server_term), + &elected_term); + /* start the server if we were asked to */ + if (elected_term > 0) + ret = scoutfs_server_start(sb, &opts->server_addr, + elected_term); + ret = -ENOTCONN; + goto out; + } + + /* send a greeting to verify endpoints of each connection */ + greet.fsid = super->hdr.fsid; + greet.format_hash = super->format_hash; + greet.server_term = cpu_to_le64(client->server_term); + greet.unmount_barrier = cpu_to_le64(client->greeting_umb); + greet.rid = cpu_to_le64(sbi->rid); + greet.flags = 0; + if (client->sending_farewell) + greet.flags |= cpu_to_le64(SCOUTFS_NET_GREETING_FLAG_FAREWELL); + if (am_voter) + greet.flags |= cpu_to_le64(SCOUTFS_NET_GREETING_FLAG_VOTER); + + ret = scoutfs_net_submit_request(sb, client->conn, + SCOUTFS_NET_CMD_GREETING, + &greet, sizeof(greet), + client_greeting, NULL, NULL); + if (ret) + scoutfs_net_shutdown(sb, client->conn); +out: + kfree(super); + + /* always have a small delay before retrying to avoid storms */ + if (ret && !atomic_read(&client->shutting_down)) + queue_delayed_work(client->workq, &client->connect_dwork, + msecs_to_jiffies(CLIENT_CONNECT_DELAY_MS)); +} + +static scoutfs_net_request_t client_req_funcs[] = { + [SCOUTFS_NET_CMD_LOCK] = client_lock, + [SCOUTFS_NET_CMD_LOCK_RECOVER] = client_lock_recover, +}; + +/* + * Called when either a connect attempt or established connection times + * out and fails. + */ +static void client_notify_down(struct super_block *sb, + struct scoutfs_net_connection *conn, void *info, + u64 rid) +{ + struct client_info *client = SCOUTFS_SB(sb)->client_info; + + if (!atomic_read(&client->shutting_down)) + queue_delayed_work(client->workq, &client->connect_dwork, 0); +} + +int scoutfs_client_setup(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct client_info *client; + int ret; + + client = kzalloc(sizeof(struct client_info), GFP_KERNEL); + if (!client) { + ret = -ENOMEM; + goto out; + } + sbi->client_info = client; + + client->sb = sb; + atomic_set(&client->shutting_down, 0); + INIT_DELAYED_WORK(&client->connect_dwork, + scoutfs_client_connect_worker); + init_completion(&client->farewell_comp); + + client->conn = scoutfs_net_alloc_conn(sb, NULL, client_notify_down, 0, + client_req_funcs, "client"); + if (!client->conn) { + ret = -ENOMEM; + goto out; + } + + client->workq = alloc_workqueue("scoutfs_client_workq", WQ_UNBOUND, 1); + if (!client->workq) { + ret = -ENOMEM; + goto out; + } + + queue_delayed_work(client->workq, &client->connect_dwork, 0); + ret = 0; + +out: + if (ret) + scoutfs_client_destroy(sb); + return ret; +} + +/* Once we get a response from the server we can shut down */ +static int client_farewell_response(struct super_block *sb, + struct scoutfs_net_connection *conn, + void *resp, unsigned int resp_len, + int error, void *data) +{ + struct client_info *client = SCOUTFS_SB(sb)->client_info; + + if (resp_len != 0) + return -EINVAL; + + client->farewell_error = error; + complete(&client->farewell_comp); + + return 0; +} + +/* + * There must be no more callers to the client request functions by the + * time we get here. + * + * If we've connected to a server then we send them a farewell request + * so that they don't wait for us to reconnect and trigger a timeout. + * + * This decision is a little racy. The server considers us connected + * when it records a persistent record of our rid as it processes our + * greeting. We can disconnect before receiving the greeting response + * and leave without sending a farewell. So given that awkward initial + * race, we also have a bit of a race where we just test the server_term + * to see if we've ever gotten a greeting reply from any server. We + * don't try to synchronize with pending connection attempts. + * + * The consequences of aborting a mount at just the wrong time and + * disconnecting without the farewell handshake depend on what the + * server does to timed out clients. At best it'll spit out a warning + * message that a client disconnected but it won't fence us if we didn't + * have any persistent state. + */ +void scoutfs_client_destroy(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct client_info *client = SCOUTFS_SB(sb)->client_info; + struct scoutfs_net_connection *conn; + int ret; + + if (client == NULL) + return; + + if (client->server_term != 0) { + client->sending_farewell = true; + ret = scoutfs_net_submit_request(sb, client->conn, + SCOUTFS_NET_CMD_FAREWELL, + NULL, 0, + client_farewell_response, + NULL, NULL); + if (ret == 0) { + ret = wait_for_completion_interruptible( + &client->farewell_comp); + if (ret == 0) + ret = client->farewell_error; + } + if (ret) { + scoutfs_inc_counter(sb, client_farewell_error); + scoutfs_warn(sb, "client saw farewell error %d, server might see client connection time out", ret); + } + } + + /* stop notify_down from queueing connect work */ + atomic_set(&client->shutting_down, 1); + + /* make sure worker isn't using the conn */ + cancel_delayed_work_sync(&client->connect_dwork); + + /* make racing conn use explode */ + conn = client->conn; + client->conn = NULL; + scoutfs_net_free_conn(sb, conn); + + if (client->workq) + destroy_workqueue(client->workq); + kfree(client); + sbi->client_info = NULL; +} diff --git a/kmod/src/client.h b/kmod/src/client.h new file mode 100644 index 00000000..ae830ef8 --- /dev/null +++ b/kmod/src/client.h @@ -0,0 +1,29 @@ +#ifndef _SCOUTFS_CLIENT_H_ +#define _SCOUTFS_CLIENT_H_ + +int scoutfs_client_alloc_inodes(struct super_block *sb, u64 count, + u64 *ino, u64 *nr); +int scoutfs_client_get_log_trees(struct super_block *sb, + struct scoutfs_log_trees *lt); +int scoutfs_client_commit_log_trees(struct super_block *sb, + struct scoutfs_log_trees *lt); +int scoutfs_client_get_roots(struct super_block *sb, + struct scoutfs_net_roots *roots); +u64 *scoutfs_client_bulk_alloc(struct super_block *sb); +int scoutfs_client_advance_seq(struct super_block *sb, u64 *seq); +int scoutfs_client_get_last_seq(struct super_block *sb, u64 *seq); +int scoutfs_client_lock_request(struct super_block *sb, + struct scoutfs_net_lock *nl); +int scoutfs_client_lock_response(struct super_block *sb, u64 net_id, + struct scoutfs_net_lock *nl); +int scoutfs_client_lock_recover_response(struct super_block *sb, u64 net_id, + struct scoutfs_net_lock_recover *nlr); +int scoutfs_client_srch_get_compact(struct super_block *sb, + struct scoutfs_srch_compact *sc); +int scoutfs_client_srch_commit_compact(struct super_block *sb, + struct scoutfs_srch_compact *res); + +int scoutfs_client_setup(struct super_block *sb); +void scoutfs_client_destroy(struct super_block *sb); + +#endif diff --git a/kmod/src/cmp.h b/kmod/src/cmp.h new file mode 100644 index 00000000..23c6d8a6 --- /dev/null +++ b/kmod/src/cmp.h @@ -0,0 +1,23 @@ +#ifndef _SCOUTFS_CMP_H_ +#define _SCOUTFS_CMP_H_ + +/* + * A generic ternary comparison macro with strict type checking. + */ +#define scoutfs_cmp(a, b) \ +({ \ + __typeof__(a) _a = (a); \ + __typeof__(b) _b = (b); \ + int _ret; \ + \ + (void) (&_a == &_b); \ + _ret = _a < _b ? -1 : _a > _b ? 1 : 0; \ + _ret; \ +}) + +static inline int scoutfs_cmp_u64s(u64 a, u64 b) +{ + return a < b ? -1 : a > b ? 1 : 0; +} + +#endif diff --git a/kmod/src/count.h b/kmod/src/count.h new file mode 100644 index 00000000..5756f407 --- /dev/null +++ b/kmod/src/count.h @@ -0,0 +1,315 @@ +#ifndef _SCOUTFS_COUNT_H_ +#define _SCOUTFS_COUNT_H_ + +/* + * Our estimate of the space consumed while dirtying items is based on + * the number of items and the size of their values. + * + * The estimate is still a read-only input to entering the transaction. + * We'd like to use it as a clean rhs arg to hold_trans. We define SIC_ + * functions which return the count struct. This lets us have a single + * arg and avoid bugs in initializing and passing in struct pointers + * from callers. The internal __count functions are used compose an + * estimate out of the sets of items it manipulates. We program in much + * clearer C instead of in the preprocessor. + * + * Compilers are able to collapse the inlines into constants for the + * constant estimates. + */ + +struct scoutfs_item_count { + signed items; + signed vals; +}; + +/* The caller knows exactly what they're doing. */ +static inline const struct scoutfs_item_count SIC_EXACT(signed items, + signed vals) +{ + struct scoutfs_item_count cnt = { + .items = items, + .vals = vals, + }; + + return cnt; +} + +/* + * Allocating an inode creates a new set of indexed items. + */ +static inline void __count_alloc_inode(struct scoutfs_item_count *cnt) +{ + const int nr_indices = SCOUTFS_INODE_INDEX_NR; + + cnt->items += 1 + nr_indices; + cnt->vals += sizeof(struct scoutfs_inode); +} + +/* + * Dirtying an inode dirties the inode item and can delete and create + * the full set of indexed items. + */ +static inline void __count_dirty_inode(struct scoutfs_item_count *cnt) +{ + const int nr_indices = 2 * SCOUTFS_INODE_INDEX_NR; + + cnt->items += 1 + nr_indices; + cnt->vals += sizeof(struct scoutfs_inode); +} + +static inline const struct scoutfs_item_count SIC_ALLOC_INODE(void) +{ + struct scoutfs_item_count cnt = {0,}; + + __count_alloc_inode(&cnt); + + return cnt; +} + +static inline const struct scoutfs_item_count SIC_DIRTY_INODE(void) +{ + struct scoutfs_item_count cnt = {0,}; + + __count_dirty_inode(&cnt); + + return cnt; +} + +/* + * Directory entries are stored in three items. + */ +static inline void __count_dirents(struct scoutfs_item_count *cnt, + unsigned name_len) +{ + cnt->items += 3; + cnt->vals += 3 * offsetof(struct scoutfs_dirent, name[name_len]); +} + +static inline void __count_sym_target(struct scoutfs_item_count *cnt, + unsigned size) +{ + unsigned nr = DIV_ROUND_UP(size, SCOUTFS_MAX_VAL_SIZE); + + cnt->items += nr; + cnt->vals += size; +} + +static inline void __count_orphan(struct scoutfs_item_count *cnt) +{ + + cnt->items += 1; +} + +static inline void __count_mknod(struct scoutfs_item_count *cnt, + unsigned name_len) +{ + __count_alloc_inode(cnt); + __count_dirents(cnt, name_len); + __count_dirty_inode(cnt); +} + +static inline const struct scoutfs_item_count SIC_MKNOD(unsigned name_len) +{ + struct scoutfs_item_count cnt = {0,}; + + __count_mknod(&cnt, name_len); + + return cnt; +} + +/* + * Dropping the inode deletes all its items. Potentially enormous numbers + * of items (data mapping, xattrs) are deleted in their own transactions. + */ +static inline const struct scoutfs_item_count SIC_DROP_INODE(int mode, + u64 size) +{ + struct scoutfs_item_count cnt = {0,}; + + if (S_ISLNK(mode)) + __count_sym_target(&cnt, size); + __count_dirty_inode(&cnt); + __count_orphan(&cnt); + + cnt.vals = 0; + return cnt; +} + +static inline const struct scoutfs_item_count SIC_LINK(unsigned name_len) +{ + struct scoutfs_item_count cnt = {0,}; + + __count_dirents(&cnt, name_len); + __count_dirty_inode(&cnt); + __count_dirty_inode(&cnt); + + return cnt; +} + +/* + * Unlink can add orphan items. + */ +static inline const struct scoutfs_item_count SIC_UNLINK(unsigned name_len) +{ + struct scoutfs_item_count cnt = {0,}; + + __count_dirents(&cnt, name_len); + __count_dirty_inode(&cnt); + __count_dirty_inode(&cnt); + __count_orphan(&cnt); + + return cnt; +} + +static inline const struct scoutfs_item_count SIC_SYMLINK(unsigned name_len, + unsigned size) +{ + struct scoutfs_item_count cnt = {0,}; + + __count_mknod(&cnt, name_len); + __count_sym_target(&cnt, size); + + return cnt; +} + +/* + * This assumes the worst case of a rename between directories that + * unlinks an existing target. That'll be worse than the common case + * by a few hundred bytes. + */ +static inline const struct scoutfs_item_count SIC_RENAME(unsigned old_len, + unsigned new_len) +{ + struct scoutfs_item_count cnt = {0,}; + + /* dirty dirs and inodes */ + __count_dirty_inode(&cnt); + __count_dirty_inode(&cnt); + __count_dirty_inode(&cnt); + __count_dirty_inode(&cnt); + + /* unlink old and new, link new */ + __count_dirents(&cnt, old_len); + __count_dirents(&cnt, new_len); + __count_dirents(&cnt, new_len); + + /* orphan the existing target */ + __count_orphan(&cnt); + + return cnt; +} + +/* + * Creating an xattr results in a dirty set of items with values that + * store the xattr header, name, and value. There's always at least one + * item with the header and name. Any previously existing items are + * deleted which dirties their key but removes their value. The two + * sets of items are indexed by different ids so their items don't + * overlap. + */ +static inline const struct scoutfs_item_count SIC_XATTR_SET(unsigned old_parts, + bool creating, + unsigned name_len, + unsigned size) +{ + struct scoutfs_item_count cnt = {0,}; + unsigned int new_parts; + + __count_dirty_inode(&cnt); + + if (old_parts) + cnt.items += old_parts; + + if (creating) { + new_parts = SCOUTFS_XATTR_NR_PARTS(name_len, size); + + cnt.items += new_parts; + cnt.vals += sizeof(struct scoutfs_xattr) + name_len + size; + } + + return cnt; +} + +/* + * write_begin can have to allocate all the blocks in the page and can + * have to add a big allocation from the server to do so: + * - merge added free extents from the server + * - remove a free extent per block + * - remove an offline extent for every other block + * - add a file extent per block + */ +static inline const struct scoutfs_item_count SIC_WRITE_BEGIN(void) +{ + struct scoutfs_item_count cnt = {0,}; + unsigned nr_free = (1 + SCOUTFS_BLOCK_SM_PER_PAGE) * 3; + unsigned nr_file = (DIV_ROUND_UP(SCOUTFS_BLOCK_SM_PER_PAGE, 2) + + SCOUTFS_BLOCK_SM_PER_PAGE) * 3; + + __count_dirty_inode(&cnt); + + cnt.items += nr_free + nr_file; + cnt.vals += nr_file; + + return cnt; +} + +/* + * Truncating an extent can: + * - delete existing file extent, + * - create two surrounding file extents, + * - add an offline file extent, + * - delete two existing free extents + * - create a merged free extent + */ +static inline const struct scoutfs_item_count +SIC_TRUNC_EXTENT(struct inode *inode) +{ + struct scoutfs_item_count cnt = {0,}; + unsigned int nr_file = 1 + 2 + 1; + unsigned int nr_free = (2 + 1) * 2; + + if (inode) + __count_dirty_inode(&cnt); + + cnt.items += nr_file + nr_free; + cnt.vals += nr_file; + + return cnt; +} + +/* + * Fallocating an extent can, at most: + * - allocate from the server: delete two free and insert merged + * - free an allocated extent: delete one and create two split + * - remove an unallocated file extent: delete one and create two split + * - add an fallocated flie extent: delete two and inset one merged + */ +static inline const struct scoutfs_item_count SIC_FALLOCATE_ONE(void) +{ + struct scoutfs_item_count cnt = {0,}; + unsigned int nr_free = ((1 + 2) * 2) * 2; + unsigned int nr_file = (1 + 2) * 2; + + __count_dirty_inode(&cnt); + + cnt.items += nr_free + nr_file; + cnt.vals += nr_file; + + return cnt; +} + +/* + * ioc_setattr_more can dirty the inode and add a single offline extent. + */ +static inline const struct scoutfs_item_count SIC_SETATTR_MORE(void) +{ + struct scoutfs_item_count cnt = {0,}; + + __count_dirty_inode(&cnt); + + cnt.items++; + + return cnt; +} + +#endif diff --git a/kmod/src/counters.c b/kmod/src/counters.c new file mode 100644 index 00000000..5578ae26 --- /dev/null +++ b/kmod/src/counters.c @@ -0,0 +1,132 @@ +/* + * Copyright (C) 2016 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include +#include + +#include "super.h" +#include "sysfs.h" +#include "counters.h" + +/* + * Maintain simple percpu counters which are always ticking. sysfs + * makes this a whole lot more noisy than it needs to be. + */ + +#undef EXPAND_COUNTER +#define EXPAND_COUNTER(which) { .name = __stringify(which), .mode = 0644 }, +static struct attribute scoutfs_counter_attrs[] = { + EXPAND_EACH_COUNTER +}; + +/* zero BSS and + 1 makes this null terminated */ +#define NR_ATTRS ARRAY_SIZE(scoutfs_counter_attrs) +static struct attribute *scoutfs_counter_attr_ptrs[NR_ATTRS + 1]; + +static ssize_t scoutfs_counter_attr_show(struct kobject *kobj, + struct attribute *attr, char *buf) +{ + struct scoutfs_counters *counters; + struct percpu_counter *pcpu; + size_t index; + + /* use the index in the _attrs array to discover the pcpu pointer */ + counters = container_of(kobj, struct scoutfs_counters, kobj); + index = attr - scoutfs_counter_attrs; + pcpu = &counters->FIRST_COUNTER + index; + + return snprintf(buf, PAGE_SIZE, "%lld\n", percpu_counter_sum(pcpu)); +} + +static void scoutfs_counters_kobj_release(struct kobject *kobj) +{ + struct scoutfs_counters *counters; + + counters = container_of(kobj, struct scoutfs_counters, kobj); + + complete(&counters->comp); +} + +static const struct sysfs_ops scoutfs_counter_attr_ops = { + .show = scoutfs_counter_attr_show, +}; + +static struct kobj_type scoutfs_counters_ktype = { + .default_attrs = scoutfs_counter_attr_ptrs, + .sysfs_ops = &scoutfs_counter_attr_ops, + .release = scoutfs_counters_kobj_release, +}; + +int scoutfs_setup_counters(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_counters *counters; + struct percpu_counter *pcpu; + int ret; + + counters = kzalloc(sizeof(struct scoutfs_counters), GFP_KERNEL); + if (!counters) + return -ENOMEM; + sbi->counters = counters; + + scoutfs_foreach_counter(sb, pcpu) { + ret = percpu_counter_init(pcpu, 0, GFP_KERNEL); + if (ret) + goto out; + } + + init_completion(&counters->comp); + ret = kobject_init_and_add(&counters->kobj, &scoutfs_counters_ktype, + scoutfs_sysfs_sb_dir(sb), "counters"); +out: + if (ret) { + /* tear down partial to avoid destroying null kobjs */ + scoutfs_foreach_counter(sb, pcpu) + percpu_counter_destroy(pcpu); + kfree(counters); + sbi->counters = NULL; + } + + return ret; +} + +void scoutfs_destroy_counters(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_counters *counters = sbi->counters; + struct percpu_counter *pcpu; + + /* this only destroys fully initialized counters */ + if (!counters) + return; + + kobject_del(&counters->kobj); + kobject_put(&counters->kobj); + wait_for_completion(&counters->comp); + + scoutfs_foreach_counter(sb, pcpu) + percpu_counter_destroy(pcpu); + + kfree(counters); + sbi->counters = NULL; +} + +void __init scoutfs_init_counters(void) +{ + int i; + + /* not ARRAY_SIZE because that would clobber null term */ + for (i = 0; i < NR_ATTRS; i++) + scoutfs_counter_attr_ptrs[i] = &scoutfs_counter_attrs[i]; +} diff --git a/kmod/src/counters.h b/kmod/src/counters.h new file mode 100644 index 00000000..6c470e58 --- /dev/null +++ b/kmod/src/counters.h @@ -0,0 +1,220 @@ +#ifndef _SCOUTFS_COUNTERS_H_ +#define _SCOUTFS_COUNTERS_H_ + +#include +#include +#include + +#include "super.h" + +/* + * We only have to define each counter here and it'll be enumerated in + * other places by this macro. Don't forget to update LAST_COUNTER. + */ +#define EXPAND_EACH_COUNTER \ + EXPAND_COUNTER(alloc_alloc_data) \ + EXPAND_COUNTER(alloc_alloc_meta) \ + EXPAND_COUNTER(alloc_free_data) \ + EXPAND_COUNTER(alloc_free_meta) \ + EXPAND_COUNTER(alloc_list_avail_lo) \ + EXPAND_COUNTER(alloc_list_freed_hi) \ + EXPAND_COUNTER(alloc_move) \ + EXPAND_COUNTER(alloc_moved_extent) \ + EXPAND_COUNTER(alloc_stale_cached_list_block) \ + EXPAND_COUNTER(block_cache_access) \ + EXPAND_COUNTER(block_cache_alloc_failure) \ + EXPAND_COUNTER(block_cache_alloc_page_order) \ + EXPAND_COUNTER(block_cache_alloc_virt) \ + EXPAND_COUNTER(block_cache_end_io_error) \ + EXPAND_COUNTER(block_cache_forget) \ + EXPAND_COUNTER(block_cache_free) \ + EXPAND_COUNTER(block_cache_invalidate) \ + EXPAND_COUNTER(block_cache_lru_move) \ + EXPAND_COUNTER(block_cache_shrink) \ + EXPAND_COUNTER(btree_compact_values) \ + EXPAND_COUNTER(btree_compact_values_enomem) \ + EXPAND_COUNTER(btree_delete) \ + EXPAND_COUNTER(btree_dirty) \ + EXPAND_COUNTER(btree_force) \ + EXPAND_COUNTER(btree_join) \ + EXPAND_COUNTER(btree_insert) \ + EXPAND_COUNTER(btree_leaf_item_hash_search) \ + EXPAND_COUNTER(btree_lookup) \ + EXPAND_COUNTER(btree_next) \ + EXPAND_COUNTER(btree_prev) \ + EXPAND_COUNTER(btree_read_error) \ + EXPAND_COUNTER(btree_split) \ + EXPAND_COUNTER(btree_stale_read) \ + EXPAND_COUNTER(btree_update) \ + EXPAND_COUNTER(btree_walk) \ + EXPAND_COUNTER(btree_walk_restart) \ + EXPAND_COUNTER(client_farewell_error) \ + EXPAND_COUNTER(corrupt_btree_block_level) \ + EXPAND_COUNTER(corrupt_btree_no_child_ref) \ + EXPAND_COUNTER(corrupt_dirent_backref_name_len) \ + EXPAND_COUNTER(corrupt_dirent_name_len) \ + EXPAND_COUNTER(corrupt_dirent_readdir_name_len) \ + EXPAND_COUNTER(corrupt_inode_block_counts) \ + EXPAND_COUNTER(corrupt_symlink_inode_size) \ + EXPAND_COUNTER(corrupt_symlink_missing_item) \ + EXPAND_COUNTER(corrupt_symlink_not_null_term) \ + EXPAND_COUNTER(dentry_revalidate_error) \ + EXPAND_COUNTER(dentry_revalidate_invalid) \ + EXPAND_COUNTER(dentry_revalidate_locked) \ + EXPAND_COUNTER(dentry_revalidate_orphan) \ + EXPAND_COUNTER(dentry_revalidate_rcu) \ + EXPAND_COUNTER(dentry_revalidate_root) \ + EXPAND_COUNTER(dentry_revalidate_valid) \ + EXPAND_COUNTER(dir_backref_excessive_retries) \ + EXPAND_COUNTER(ext_op_insert) \ + EXPAND_COUNTER(ext_op_next) \ + EXPAND_COUNTER(ext_op_remove) \ + EXPAND_COUNTER(forest_bloom_fail) \ + EXPAND_COUNTER(forest_bloom_pass) \ + EXPAND_COUNTER(forest_read_items) \ + EXPAND_COUNTER(forest_roots_next_hint) \ + EXPAND_COUNTER(forest_set_bloom_bits) \ + EXPAND_COUNTER(item_clear_dirty) \ + EXPAND_COUNTER(item_create) \ + EXPAND_COUNTER(item_delete) \ + EXPAND_COUNTER(item_dirty) \ + EXPAND_COUNTER(item_invalidate) \ + EXPAND_COUNTER(item_invalidate_page) \ + EXPAND_COUNTER(item_lookup) \ + EXPAND_COUNTER(item_mark_dirty) \ + EXPAND_COUNTER(item_next) \ + EXPAND_COUNTER(item_page_accessed) \ + EXPAND_COUNTER(item_page_alloc) \ + EXPAND_COUNTER(item_page_clear_dirty) \ + EXPAND_COUNTER(item_page_compact) \ + EXPAND_COUNTER(item_page_free) \ + EXPAND_COUNTER(item_page_lru_add) \ + EXPAND_COUNTER(item_page_lru_remove) \ + EXPAND_COUNTER(item_page_mark_dirty) \ + EXPAND_COUNTER(item_page_rbtree_walk) \ + EXPAND_COUNTER(item_page_split) \ + EXPAND_COUNTER(item_pcpu_add_replaced) \ + EXPAND_COUNTER(item_pcpu_page_hit) \ + EXPAND_COUNTER(item_pcpu_page_miss) \ + EXPAND_COUNTER(item_pcpu_page_miss_keys) \ + EXPAND_COUNTER(item_read_pages_split) \ + EXPAND_COUNTER(item_shrink_page) \ + EXPAND_COUNTER(item_shrink_page_dirty) \ + EXPAND_COUNTER(item_shrink_page_reader) \ + EXPAND_COUNTER(item_shrink_page_trylock) \ + EXPAND_COUNTER(item_update) \ + EXPAND_COUNTER(item_write_dirty) \ + EXPAND_COUNTER(lock_alloc) \ + EXPAND_COUNTER(lock_free) \ + EXPAND_COUNTER(lock_grace_extended) \ + EXPAND_COUNTER(lock_grace_set) \ + EXPAND_COUNTER(lock_grace_wait) \ + EXPAND_COUNTER(lock_grant_request) \ + EXPAND_COUNTER(lock_grant_response) \ + EXPAND_COUNTER(lock_grant_work) \ + EXPAND_COUNTER(lock_invalidate_coverage) \ + EXPAND_COUNTER(lock_invalidate_inode) \ + EXPAND_COUNTER(lock_invalidate_request) \ + EXPAND_COUNTER(lock_invalidate_response) \ + EXPAND_COUNTER(lock_invalidate_sync) \ + EXPAND_COUNTER(lock_invalidate_work) \ + EXPAND_COUNTER(lock_lock) \ + EXPAND_COUNTER(lock_lock_error) \ + EXPAND_COUNTER(lock_nonblock_eagain) \ + EXPAND_COUNTER(lock_recover_request) \ + EXPAND_COUNTER(lock_shrink_attempted) \ + EXPAND_COUNTER(lock_shrink_aborted) \ + EXPAND_COUNTER(lock_shrink_work) \ + EXPAND_COUNTER(lock_unlock) \ + EXPAND_COUNTER(lock_wait) \ + EXPAND_COUNTER(net_dropped_response) \ + EXPAND_COUNTER(net_send_bytes) \ + EXPAND_COUNTER(net_send_error) \ + EXPAND_COUNTER(net_send_messages) \ + EXPAND_COUNTER(net_recv_bytes) \ + EXPAND_COUNTER(net_recv_dropped_duplicate) \ + EXPAND_COUNTER(net_recv_error) \ + EXPAND_COUNTER(net_recv_invalid_message) \ + EXPAND_COUNTER(net_recv_messages) \ + EXPAND_COUNTER(net_unknown_request) \ + EXPAND_COUNTER(quorum_cycle) \ + EXPAND_COUNTER(quorum_elected_leader) \ + EXPAND_COUNTER(quorum_election_timeout) \ + EXPAND_COUNTER(quorum_failure) \ + EXPAND_COUNTER(quorum_read_block) \ + EXPAND_COUNTER(quorum_read_block_error) \ + EXPAND_COUNTER(quorum_read_invalid_block) \ + EXPAND_COUNTER(quorum_saw_super_leader) \ + EXPAND_COUNTER(quorum_timedout) \ + EXPAND_COUNTER(quorum_write_block) \ + EXPAND_COUNTER(quorum_write_block_error) \ + EXPAND_COUNTER(quorum_fenced) \ + EXPAND_COUNTER(server_commit_hold) \ + EXPAND_COUNTER(server_commit_queue) \ + EXPAND_COUNTER(server_commit_worker) \ + EXPAND_COUNTER(srch_add_entry) \ + EXPAND_COUNTER(srch_compact_dirty_block) \ + EXPAND_COUNTER(srch_compact_entry) \ + EXPAND_COUNTER(srch_compact_flush) \ + EXPAND_COUNTER(srch_compact_log_page) \ + EXPAND_COUNTER(srch_compact_removed_entry) \ + EXPAND_COUNTER(srch_inconsistent_ref) \ + EXPAND_COUNTER(srch_rotate_log) \ + EXPAND_COUNTER(srch_search_log) \ + EXPAND_COUNTER(srch_search_log_block) \ + EXPAND_COUNTER(srch_search_retry_empty) \ + EXPAND_COUNTER(srch_search_sorted) \ + EXPAND_COUNTER(srch_search_sorted_block) \ + EXPAND_COUNTER(srch_search_stale_eio) \ + EXPAND_COUNTER(srch_search_stale_retry) \ + EXPAND_COUNTER(srch_search_xattrs) \ + EXPAND_COUNTER(srch_read_stale) \ + EXPAND_COUNTER(statfs) \ + EXPAND_COUNTER(trans_commit_data_alloc_low) \ + EXPAND_COUNTER(trans_commit_dirty_meta_full) \ + EXPAND_COUNTER(trans_commit_fsync) \ + EXPAND_COUNTER(trans_commit_meta_alloc_low) \ + EXPAND_COUNTER(trans_commit_sync_fs) \ + EXPAND_COUNTER(trans_commit_timer) \ + EXPAND_COUNTER(trans_commit_written) + +#define FIRST_COUNTER alloc_alloc_data +#define LAST_COUNTER trans_commit_written + +#undef EXPAND_COUNTER +#define EXPAND_COUNTER(which) struct percpu_counter which; + +struct scoutfs_counters { + /* $sysfs/fs/scoutfs/$id/counters/ */ + struct kobject kobj; + struct completion comp; + + EXPAND_EACH_COUNTER +}; + +#define scoutfs_foreach_counter(sb, pcpu) \ + for (pcpu = &SCOUTFS_SB(sb)->counters->FIRST_COUNTER; \ + pcpu <= &SCOUTFS_SB(sb)->counters->LAST_COUNTER; \ + pcpu++) + +/* + * We always read with _sum, we have no use for the shared count and + * certainly don't want to pay the cost of a shared lock to update it. + * The default batch of 32 make counter increments show up significantly + * in profiles. + */ +#define SCOUTFS_PCPU_COUNTER_BATCH (1 << 30) + +#define scoutfs_inc_counter(sb, which) \ + __percpu_counter_add(&SCOUTFS_SB(sb)->counters->which, 1, \ + SCOUTFS_PCPU_COUNTER_BATCH) + +#define scoutfs_add_counter(sb, which, cnt) \ + __percpu_counter_add(&SCOUTFS_SB(sb)->counters->which, cnt, \ + SCOUTFS_PCPU_COUNTER_BATCH) + +void __init scoutfs_init_counters(void); +int scoutfs_setup_counters(struct super_block *sb); +void scoutfs_destroy_counters(struct super_block *sb); + +#endif diff --git a/kmod/src/data.c b/kmod/src/data.c new file mode 100644 index 00000000..a8bca721 --- /dev/null +++ b/kmod/src/data.c @@ -0,0 +1,1545 @@ +/* + * Copyright (C) 2019 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "format.h" +#include "super.h" +#include "inode.h" +#include "key.h" +#include "alloc.h" +#include "data.h" +#include "trans.h" +#include "counters.h" +#include "scoutfs_trace.h" +#include "item.h" +#include "ioctl.h" +#include "btree.h" +#include "lock.h" +#include "file.h" +#include "msg.h" +#include "count.h" +#include "ext.h" + +/* + * We want to amortize work done after dirtying the shared transaction + * accounting, but we don't want to blow out dirty allocator btree + * blocks. Each allocation can dirty quite a few allocator btree blocks + * so we check in pretty often. + */ +#define EXTENTS_PER_HOLD 8 + +struct data_info { + struct super_block *sb; + struct mutex mutex; + struct scoutfs_alloc *alloc; + struct scoutfs_block_writer *wri; + struct scoutfs_alloc_root data_avail; + struct scoutfs_alloc_root data_freed; + struct scoutfs_extent cached_ext; +}; + +#define DECLARE_DATA_INFO(sb, name) \ + struct data_info *name = SCOUTFS_SB(sb)->data_info + +struct data_ext_args { + u64 ino; + struct inode *inode; + struct scoutfs_lock *lock; +}; + +static void item_from_extent(struct scoutfs_key *key, + struct scoutfs_data_extent_val *dv, u64 ino, + u64 start, u64 len, u64 map, u8 flags) +{ + *key = (struct scoutfs_key) { + .sk_zone = SCOUTFS_FS_ZONE, + .skdx_ino = cpu_to_le64(ino), + .sk_type = SCOUTFS_DATA_EXTENT_TYPE, + .skdx_end = cpu_to_le64(start + len - 1), + .skdx_len = cpu_to_le64(len), + }; + dv->blkno = cpu_to_le64(map); + dv->flags = flags; +} + +static void ext_from_item(struct scoutfs_extent *ext, + struct scoutfs_key *key, + struct scoutfs_data_extent_val *dv) +{ + ext->start = le64_to_cpu(key->skdx_end) - + le64_to_cpu(key->skdx_len) + 1; + ext->len = le64_to_cpu(key->skdx_len); + ext->map = le64_to_cpu(dv->blkno); + ext->flags = dv->flags; +} + +static int data_ext_next(struct super_block *sb, void *arg, u64 start, u64 len, + struct scoutfs_extent *ext) +{ + struct data_ext_args *args = arg; + struct scoutfs_data_extent_val dv; + struct scoutfs_key key; + struct scoutfs_key last; + int ret; + + item_from_extent(&last, &dv, args->ino, U64_MAX, 1, 0, 0); + item_from_extent(&key, &dv, args->ino, start, len, 0, 0); + + ret = scoutfs_item_next(sb, &key, &last, &dv, sizeof(dv), args->lock); + if (ret == sizeof(dv)) { + ext_from_item(ext, &key, &dv); + ret = 0; + } else if (ret >= 0) { + ret = -EIO; + } + + if (ret < 0) + memset(ext, 0, sizeof(struct scoutfs_extent)); + return ret; +} + +static void add_onoff(struct inode *inode, u64 map, u8 flags, s64 len) +{ + s64 on = 0; + s64 off = 0; + + if (map && !(flags & SEF_UNWRITTEN)) + on += len; + else if (flags & SEF_OFFLINE) + off += len; + + scoutfs_inode_add_onoff(inode, on, off); +} + +static int data_ext_insert(struct super_block *sb, void *arg, u64 start, + u64 len, u64 map, u8 flags) +{ + struct data_ext_args *args = arg; + struct scoutfs_data_extent_val dv; + struct scoutfs_key key; + int ret; + + item_from_extent(&key, &dv, args->ino, start, len, map, flags); + ret = scoutfs_item_create(sb, &key, &dv, sizeof(dv), args->lock); + if (ret == 0 && args->inode) + add_onoff(args->inode, map, flags, len); + return ret; +} + +static int data_ext_remove(struct super_block *sb, void *arg, u64 start, + u64 len, u64 map, u8 flags) +{ + struct data_ext_args *args = arg; + struct scoutfs_data_extent_val dv; + struct scoutfs_key key; + int ret; + + item_from_extent(&key, &dv, args->ino, start, len, map, flags); + ret = scoutfs_item_delete(sb, &key, args->lock); + if (ret == 0 && args->inode) + add_onoff(args->inode, map, flags, -len); + return ret; +} + +static struct scoutfs_ext_ops data_ext_ops = { + .next = data_ext_next, + .insert = data_ext_insert, + .remove = data_ext_remove, +}; + +/* + * Find and remove or mark offline the block mappings that intersect + * with the caller's range. The caller is responsible for transactions + * and locks. + * + * Returns: + * - -errno on errors + * - 0 if there are no more extents to stop iteration + * - +iblock of next logical block to truncate the next block from + */ +static s64 truncate_extents(struct super_block *sb, struct inode *inode, + u64 ino, u64 iblock, u64 last, bool offline, + struct scoutfs_lock *lock) +{ + DECLARE_DATA_INFO(sb, datinf); + struct data_ext_args args = { + .ino = ino, + .inode = inode, + .lock = lock, + }; + struct scoutfs_extent ext; + struct scoutfs_extent tr; + u64 offset; + s64 ret; + u8 flags; + int i; + + flags = offline ? SEF_OFFLINE : 0; + ret = 0; + + for (i = 0; iblock <= last; i++) { + if (i == EXTENTS_PER_HOLD) { + ret = iblock; + break; + } + + ret = scoutfs_ext_next(sb, &data_ext_ops, &args, + iblock, 1, &ext); + if (ret < 0) { + if (ret == -ENOENT) + ret = 0; + break; + } + + /* done if we went past the region */ + if (ext.start > last) { + ret = 0; + break; + } + + /* nothing to do when already offline and unmapped */ + if ((offline && (ext.flags & SEF_OFFLINE)) && !ext.map) { + iblock = ext.start + ext.len; + continue; + } + + iblock = max(ext.start, iblock); + offset = iblock - ext.start; + + tr.start = iblock; + tr.map = ext.map ? ext.map + offset : 0; + tr.len = min(ext.len - offset, last - iblock + 1); + tr.flags = ext.flags; + + if (tr.map) { + mutex_lock(&datinf->mutex); + ret = scoutfs_free_data(sb, datinf->alloc, + datinf->wri, + &datinf->data_freed, + tr.map, tr.len); + mutex_unlock(&datinf->mutex); + if (ret < 0) + break; + } + + trace_scoutfs_data_extent_truncated(sb, ino, &tr); + + ret = scoutfs_ext_set(sb, &data_ext_ops, &args, + tr.start, tr.len, 0, flags); + BUG_ON(ret); /* inconsistent, could prealloc items */ + + iblock += tr.len; + } + + return ret; +} + +/* + * Free blocks inside the logical block range from 'iblock' to 'last', + * inclusive. + * + * If 'offline' is given then blocks are freed an offline mapping is + * left behind. Only blocks that have been allocated can be marked + * offline. + * + * If the inode is provided then we update its tracking of the online + * and offline blocks. If it's not provided then the inode is being + * destroyed and isn't reachable, we don't need to update it. + * + * The caller is in charge of locking the inode and data, but we may + * have to modify far more items than fit in a transaction so we're in + * charge of batching updates into transactions. If the inode is + * provided then we're responsible for updating its item as we go. + */ +int scoutfs_data_truncate_items(struct super_block *sb, struct inode *inode, + u64 ino, u64 iblock, u64 last, bool offline, + struct scoutfs_lock *lock) +{ + struct scoutfs_item_count cnt = SIC_TRUNC_EXTENT(inode); + LIST_HEAD(ind_locks); + s64 ret = 0; + + WARN_ON_ONCE(inode && !mutex_is_locked(&inode->i_mutex)); + + /* clamp last to the last possible block? */ + if (last > SCOUTFS_BLOCK_SM_MAX) + last = SCOUTFS_BLOCK_SM_MAX; + + trace_scoutfs_data_truncate_items(sb, iblock, last, offline); + + if (WARN_ON_ONCE(last < iblock)) + return -EINVAL; + + while (iblock <= last) { + if (inode) + ret = scoutfs_inode_index_lock_hold(inode, &ind_locks, + true, cnt); + else + ret = scoutfs_hold_trans(sb, cnt); + if (ret) + break; + + if (inode) + ret = scoutfs_dirty_inode_item(inode, lock); + else + ret = 0; + + if (ret == 0) + ret = truncate_extents(sb, inode, ino, iblock, last, + offline, lock); + + if (inode) + scoutfs_update_inode_item(inode, lock, &ind_locks); + scoutfs_release_trans(sb); + if (inode) + scoutfs_inode_index_unlock(sb, &ind_locks); + + if (ret <= 0) + break; + + iblock = ret; + ret = 0; + } + + return ret; +} + +static inline u64 ext_last(struct scoutfs_extent *ext) +{ + return ext->start + ext->len - 1; +} + +/* + * The caller is writing to a logical iblock that doesn't have an + * allocated extent. + * + * We always allocate an extent starting at the logical iblock. The + * caller has searched for an extent containing iblock. If it already + * existed then it must be unallocated and offline. + * + * Preallocation is used if we're strictly contiguously extending + * writes. That is, if the logical block offset equals the number of + * online blocks. We try to preallocate the number of blocks existing + * so that small files don't waste inordinate amounts of space and large + * files will eventually see large extents. This only works for + * contiguous single stream writes or stages of files from the first + * block. It doesn't work for concurrent stages, releasing behind + * staging, sparse files, multi-node writes, etc. fallocate() is always + * a better tool to use. + */ +static int alloc_block(struct super_block *sb, struct inode *inode, + struct scoutfs_extent *ext, u64 iblock, + struct scoutfs_lock *lock) +{ + DECLARE_DATA_INFO(sb, datinf); + const u64 ino = scoutfs_ino(inode); + struct data_ext_args args = { + .ino = ino, + .inode = inode, + .lock = lock, + }; + struct scoutfs_extent found; + struct scoutfs_extent pre; + u64 blkno = 0; + u64 online; + u64 offline; + u8 flags; + u64 count; + int ret; + int err; + + trace_scoutfs_data_alloc_block_enter(sb, ino, iblock, ext); + + /* can only allocate over existing unallocated offline extent */ + if (WARN_ON_ONCE(ext->len && + !(iblock >= ext->start && iblock <= ext_last(ext) && + ext->map == 0 && (ext->flags & SEF_OFFLINE)))) + return -EINVAL; + + mutex_lock(&datinf->mutex); + + scoutfs_inode_get_onoff(inode, &online, &offline); + + if (ext->len) { + /* limit preallocation to remaining existing (offline) extent */ + count = ext->len - (iblock - ext->start); + flags = ext->flags; + } else { + /* otherwise alloc to next extent */ + ret = scoutfs_ext_next(sb, &data_ext_ops, &args, + iblock, 1, &found); + if (ret < 0 && ret != -ENOENT) + goto out; + if (found.len && found.start > iblock) + count = found.start - iblock; + else + count = SCOUTFS_DATA_EXTEND_PREALLOC_LIMIT; + flags = 0; + } + + /* overall prealloc limit */ + count = min_t(u64, count, SCOUTFS_DATA_EXTEND_PREALLOC_LIMIT); + + /* only strictly contiguous extending writes will try to preallocate */ + if (iblock > 1 && iblock == online) + count = min(iblock, count); + else + count = 1; + + ret = scoutfs_alloc_data(sb, datinf->alloc, datinf->wri, + &datinf->data_avail, &datinf->cached_ext, + count, &blkno, &count); + if (ret < 0) + goto out; + + ret = scoutfs_ext_set(sb, &data_ext_ops, &args, iblock, 1, blkno, 0); + if (ret < 0) + goto out; + + if (count > 1) { + pre.start = iblock + 1; + pre.len = count - 1; + pre.map = blkno + 1; + pre.flags = flags | SEF_UNWRITTEN; + ret = scoutfs_ext_set(sb, &data_ext_ops, &args, pre.start, + pre.len, pre.map, pre.flags); + if (ret < 0) { + err = scoutfs_ext_set(sb, &data_ext_ops, &args, iblock, + 1, 0, flags); + BUG_ON(err); /* couldn't restore original */ + goto out; + } + } + + /* tell the caller we have a single block, could check next? */ + ext->start = iblock; + ext->len = 1; + ext->map = blkno; + ext->flags = 0; + ret = 0; +out: + if (ret < 0 && blkno > 0) { + err = scoutfs_free_data(sb, datinf->alloc, datinf->wri, + &datinf->data_freed, blkno, count); + BUG_ON(err); /* leaked free blocks */ + } + + if (ret == 0) { + trace_scoutfs_data_alloc(sb, ino, ext); + trace_scoutfs_data_prealloc(sb, ino, &pre); + } + + mutex_unlock(&datinf->mutex); + + return ret; +} + +static int scoutfs_get_block(struct inode *inode, sector_t iblock, + struct buffer_head *bh, int create) +{ + struct scoutfs_inode_info *si = SCOUTFS_I(inode); + const u64 ino = scoutfs_ino(inode); + struct super_block *sb = inode->i_sb; + struct data_ext_args args; + struct scoutfs_lock *lock = NULL; + struct scoutfs_extent ext = {0,}; + struct scoutfs_extent un; + u64 offset; + int ret; + + WARN_ON_ONCE(create && !mutex_is_locked(&inode->i_mutex)); + + /* make sure caller holds a cluster lock */ + lock = scoutfs_per_task_get(&si->pt_data_lock); + if (WARN_ON_ONCE(!lock)) { + ret = -EINVAL; + goto out; + } + + args.ino = ino; + args.inode = inode; + args.lock = lock; + + ret = scoutfs_ext_next(sb, &data_ext_ops, &args, iblock, 1, &ext); + if (ret == -ENOENT || (ret == 0 && ext.start > iblock)) + memset(&ext, 0, sizeof(ext)); + else if (ret < 0) + goto out; + + if (ext.len) + trace_scoutfs_data_get_block_found(sb, ino, &ext); + + /* non-staging callers should have waited on offline blocks */ + if (WARN_ON_ONCE(ext.map && (ext.flags & SEF_OFFLINE) && !si->staging)){ + ret = -EIO; + goto out; + } + + /* convert unwritten to written, could be staging */ + if (create && ext.map && (ext.flags & SEF_UNWRITTEN)) { + un.start = iblock; + un.len = 1; + un.map = ext.map + (iblock - ext.start); + un.flags = ext.flags & ~(SEF_OFFLINE|SEF_UNWRITTEN); + ret = scoutfs_ext_set(sb, &data_ext_ops, &args, + un.start, un.len, un.map, un.flags); + if (ret == 0) { + ext = un; + set_buffer_new(bh); + } + goto out; + } + + /* allocate and map blocks containing our logical block */ + if (create && !ext.map) { + ret = alloc_block(sb, inode, &ext, iblock, lock); + if (ret == 0) + set_buffer_new(bh); + } else { + ret = 0; + } +out: + /* map usable extent, else leave bh unmapped for sparse reads */ + if (ret == 0 && ext.map && !(ext.flags & SEF_UNWRITTEN)) { + offset = iblock - ext.start; + map_bh(bh, inode->i_sb, ext.map + offset); + bh->b_size = min_t(u64, bh->b_size, + (ext.len - offset) << SCOUTFS_BLOCK_SM_SHIFT); + trace_scoutfs_data_get_block_mapped(sb, ino, &ext); + } + + trace_scoutfs_get_block(sb, scoutfs_ino(inode), iblock, create, + &ext, ret, bh->b_blocknr, bh->b_size); + return ret; +} + +/* + * This is almost never used. We can't block on a cluster lock while + * holding the page lock because lock invalidation gets the page lock + * while blocking locks. If a non blocking lock attempt fails we unlock + * the page and block acquiring the lock. We unlocked the page so it + * could have been truncated away, or whatever, so we return + * AOP_TRUNCATED_PAGE to have the caller try again. + * + * A similar process happens if we try to read from an offline extent + * that a caller hasn't already waited for. Instead of blocking + * acquiring the lock we block waiting for the offline extent. The page + * lock protects the page from release while we're checking and + * reading the extent. + * + * We can return errors from locking and checking offline extents. The + * page is unlocked if we return an error. + */ +static int scoutfs_readpage(struct file *file, struct page *page) +{ + struct inode *inode = file->f_inode; + struct scoutfs_inode_info *si = SCOUTFS_I(inode); + struct super_block *sb = inode->i_sb; + struct scoutfs_lock *inode_lock = NULL; + SCOUTFS_DECLARE_PER_TASK_ENTRY(pt_ent); + DECLARE_DATA_WAIT(dw); + int flags; + int ret; + + flags = SCOUTFS_LKF_REFRESH_INODE | SCOUTFS_LKF_NONBLOCK; + ret = scoutfs_lock_inode(sb, SCOUTFS_LOCK_READ, flags, inode, + &inode_lock); + if (ret < 0) { + unlock_page(page); + if (ret == -EAGAIN) { + flags &= ~SCOUTFS_LKF_NONBLOCK; + ret = scoutfs_lock_inode(sb, SCOUTFS_LOCK_READ, flags, + inode, &inode_lock); + if (ret == 0) { + scoutfs_unlock(sb, inode_lock, + SCOUTFS_LOCK_READ); + ret = AOP_TRUNCATED_PAGE; + } + } + return ret; + } + + if (scoutfs_per_task_add_excl(&si->pt_data_lock, &pt_ent, inode_lock)) { + ret = scoutfs_data_wait_check(inode, page_offset(page), + PAGE_CACHE_SIZE, SEF_OFFLINE, + SCOUTFS_IOC_DWO_READ, &dw, + inode_lock); + if (ret != 0) { + unlock_page(page); + scoutfs_per_task_del(&si->pt_data_lock, &pt_ent); + scoutfs_unlock(sb, inode_lock, SCOUTFS_LOCK_READ); + } + if (ret > 0) { + ret = scoutfs_data_wait(inode, &dw); + if (ret == 0) + ret = AOP_TRUNCATED_PAGE; + } + if (ret != 0) + return ret; + } + + ret = mpage_readpage(page, scoutfs_get_block); + + scoutfs_unlock(sb, inode_lock, SCOUTFS_LOCK_READ); + scoutfs_per_task_del(&si->pt_data_lock, &pt_ent); + + return ret; +} + +/* + * This is used for opportunistic read-ahead which can throw the pages + * away if it needs to. If the caller didn't deal with offline extents + * then we drop those pages rather than trying to wait. Whoever is + * staging offline extents should be doing it in enormous chunks so that + * read-ahead can ramp up within each staged region. The check for + * offline extents is cheap when the inode has no offline extents. + */ +static int scoutfs_readpages(struct file *file, struct address_space *mapping, + struct list_head *pages, unsigned nr_pages) +{ + struct inode *inode = file->f_inode; + struct super_block *sb = inode->i_sb; + struct scoutfs_lock *inode_lock = NULL; + struct page *page; + struct page *tmp; + int ret; + + ret = scoutfs_lock_inode(sb, SCOUTFS_LOCK_READ, + SCOUTFS_LKF_REFRESH_INODE, inode, &inode_lock); + if (ret) + goto out; + + list_for_each_entry_safe(page, tmp, pages, lru) { + ret = scoutfs_data_wait_check(inode, page_offset(page), + PAGE_CACHE_SIZE, SEF_OFFLINE, + SCOUTFS_IOC_DWO_READ, NULL, + inode_lock); + if (ret < 0) + goto out; + if (ret > 0) { + list_del(&page->lru); + page_cache_release(page); + if (--nr_pages == 0) { + ret = 0; + goto out; + } + } + } + + ret = mpage_readpages(mapping, pages, nr_pages, scoutfs_get_block); +out: + scoutfs_unlock(sb, inode_lock, SCOUTFS_LOCK_READ); + BUG_ON(!list_empty(pages)); + return ret; +} + +static int scoutfs_writepage(struct page *page, struct writeback_control *wbc) +{ + return block_write_full_page(page, scoutfs_get_block, wbc); +} + +static int scoutfs_writepages(struct address_space *mapping, + struct writeback_control *wbc) +{ + return mpage_writepages(mapping, wbc, scoutfs_get_block); +} + +/* fsdata allocated in write_begin and freed in write_end */ +struct write_begin_data { + struct list_head ind_locks; + struct scoutfs_lock *lock; +}; + +static int scoutfs_write_begin(struct file *file, + struct address_space *mapping, loff_t pos, + unsigned len, unsigned flags, + struct page **pagep, void **fsdata) +{ + struct inode *inode = mapping->host; + struct scoutfs_inode_info *si = SCOUTFS_I(inode); + struct super_block *sb = inode->i_sb; + struct write_begin_data *wbd; + u64 ind_seq; + int ret; + + trace_scoutfs_write_begin(sb, scoutfs_ino(inode), (__u64)pos, len); + + wbd = kmalloc(sizeof(struct write_begin_data), GFP_NOFS); + if (!wbd) + return -ENOMEM; + + INIT_LIST_HEAD(&wbd->ind_locks); + *fsdata = wbd; + + wbd->lock = scoutfs_per_task_get(&si->pt_data_lock); + if (WARN_ON_ONCE(!wbd->lock)) { + ret = -EINVAL; + goto out; + } + + do { + ret = scoutfs_inode_index_start(sb, &ind_seq) ?: + scoutfs_inode_index_prepare(sb, &wbd->ind_locks, inode, + true) ?: + scoutfs_inode_index_try_lock_hold(sb, &wbd->ind_locks, + ind_seq, + SIC_WRITE_BEGIN()); + } while (ret > 0); + if (ret < 0) + goto out; + + /* can't re-enter fs, have trans */ + flags |= AOP_FLAG_NOFS; + + /* generic write_end updates i_size and calls dirty_inode */ + ret = scoutfs_dirty_inode_item(inode, wbd->lock); + if (ret == 0) + ret = block_write_begin(mapping, pos, len, flags, pagep, + scoutfs_get_block); + if (ret) + scoutfs_release_trans(sb); +out: + if (ret) { + scoutfs_inode_index_unlock(sb, &wbd->ind_locks); + kfree(wbd); + } + return ret; +} + +/* kinda like __filemap_fdatawrite_range! :P */ +static int writepages_sync_none(struct address_space *mapping, loff_t start, + loff_t end) +{ + struct writeback_control wbc = { + .sync_mode = WB_SYNC_NONE, + .nr_to_write = LONG_MAX, + .range_start = start, + .range_end = end, + }; + + return mapping->a_ops->writepages(mapping, &wbc); +} + +static int scoutfs_write_end(struct file *file, struct address_space *mapping, + loff_t pos, unsigned len, unsigned copied, + struct page *page, void *fsdata) +{ + struct inode *inode = mapping->host; + struct scoutfs_inode_info *si = SCOUTFS_I(inode); + struct super_block *sb = inode->i_sb; + struct write_begin_data *wbd = fsdata; + int ret; + + trace_scoutfs_write_end(sb, scoutfs_ino(inode), page->index, (u64)pos, + len, copied); + + ret = generic_write_end(file, mapping, pos, len, copied, page, fsdata); + if (ret > 0) { + if (!si->staging) { + scoutfs_inode_set_data_seq(inode); + scoutfs_inode_inc_data_version(inode); + } + + scoutfs_update_inode_item(inode, wbd->lock, &wbd->ind_locks); + scoutfs_inode_queue_writeback(inode); + } + scoutfs_release_trans(sb); + scoutfs_inode_index_unlock(sb, &wbd->ind_locks); + kfree(wbd); + + /* + * Currently transactions are kept very simple. Only one is + * open at a time and commit excludes concurrent dirtying. It + * writes out all dirty file data during commit. This can lead + * to very long commit latencies with lots of dirty file data. + * + * This hack tries to minimize these writeback latencies while + * keeping concurrent large file strreaming writes from + * suffering too terribly. Every N bytes we kick off background + * writbeack on the previous N bytes. By the time transaction + * commit comes along it will find that dirty file blocks have + * already been written. + */ +#define BACKGROUND_WRITEBACK_BYTES (16 * 1024 * 1024) +#define BACKGROUND_WRITEBACK_MASK (BACKGROUND_WRITEBACK_BYTES - 1) + if (ret > 0 && ((pos + ret) & BACKGROUND_WRITEBACK_MASK) == 0) + writepages_sync_none(mapping, + pos + ret - BACKGROUND_WRITEBACK_BYTES, + pos + ret - 1); + + return ret; +} + +/* + * Try to allocate unwritten extents for any unallocated regions of the + * logical block extent from the caller. The caller manages locks and + * transactions. We limit ourselves to a reasonable number of extents + * before returning to open another transaction. + * + * We return an error or the number of blocks starting at iblock that + * were successfully processed. The caller will continue after those + * blocks until they reach last. + */ +static s64 fallocate_extents(struct super_block *sb, struct inode *inode, + u64 iblock, u64 last, struct scoutfs_lock *lock) +{ + DECLARE_DATA_INFO(sb, datinf); + struct data_ext_args args = { + .ino = scoutfs_ino(inode), + .inode = inode, + .lock = lock, + }; + struct scoutfs_extent ext; + u8 ext_fl; + u64 blkno; + u64 count; + s64 done = 0; + int ret = 0; + int err; + int i; + + for (i = 0; iblock <= last && i < EXTENTS_PER_HOLD; i++) { + + ret = scoutfs_ext_next(sb, &data_ext_ops, &args, + iblock, 1, &ext); + if (ret == -ENOENT) + ret = 0; + else if (ret < 0) + break; + + /* default to allocate to end of region */ + count = last - iblock + 1; + ext_fl = 0; + + if (!ext.len) { + /* no extent, default alloc from above */ + + } else if (ext.start <= iblock && ext.map) { + /* skip portion of allocated extent */ + count = min_t(u64, count, + ext.len - (iblock - ext.start)); + iblock += count; + done += count; + continue; + + } else if (ext.start <= iblock && !ext.map) { + /* alloc portion of unallocated extent */ + count = min_t(u64, count, + ext.len - (iblock - ext.start)); + ext_fl = ext.flags; + + } else if (iblock < ext.start) { + /* alloc hole until next extent */ + count = min_t(u64, count, ext.start - iblock); + } + + /* limit allocation attempts */ + count = min_t(u64, count, SCOUTFS_FALLOCATE_ALLOC_LIMIT); + + mutex_lock(&datinf->mutex); + + ret = scoutfs_alloc_data(sb, datinf->alloc, datinf->wri, + &datinf->data_avail, + &datinf->cached_ext, + count, &blkno, &count); + if (ret == 0) { + ret = scoutfs_ext_set(sb, &data_ext_ops, &args, iblock, + count, blkno, + ext_fl | SEF_UNWRITTEN); + if (ret < 0) { + err = scoutfs_free_data(sb, datinf->alloc, + datinf->wri, + &datinf->data_avail, + blkno, count); + BUG_ON(err); /* inconsistent */ + } + } + + mutex_unlock(&datinf->mutex); + + if (ret < 0) + break; + + iblock += count; + done += count; + } + + if (ret == 0) + ret = done; + + return ret; +} + +/* + * Modify the extents that map the blocks that store the len byte region + * starting at offset. + * + * The caller has only prevented freezing by entering a fs write + * context. We're responsible for all other locking and consistency. + * + * This can be used to preallocate files for staging. We find existing + * offline extents and allocate block for them and set unwritten. + */ +long scoutfs_fallocate(struct file *file, int mode, loff_t offset, loff_t len) +{ + struct inode *inode = file_inode(file); + struct super_block *sb = inode->i_sb; + const u64 ino = scoutfs_ino(inode); + struct scoutfs_lock *lock = NULL; + LIST_HEAD(ind_locks); + loff_t end; + u64 iblock; + u64 last; + s64 ret; + + mutex_lock(&inode->i_mutex); + + /* XXX support more flags */ + if (mode & ~(FALLOC_FL_KEEP_SIZE)) { + ret = -EOPNOTSUPP; + goto out; + } + + /* catch wrapping */ + if (offset + len < offset) { + ret = -EINVAL; + goto out; + } + + if (len == 0) { + ret = 0; + goto out; + } + + ret = scoutfs_lock_inode(sb, SCOUTFS_LOCK_WRITE, + SCOUTFS_LKF_REFRESH_INODE, inode, &lock); + if (ret) + goto out; + + inode_dio_wait(inode); + + if (!(mode & FALLOC_FL_KEEP_SIZE) && + (offset + len > i_size_read(inode))) { + ret = inode_newsize_ok(inode, offset + len); + if (ret) + goto out; + } + + iblock = offset >> SCOUTFS_BLOCK_SM_SHIFT; + last = (offset + len - 1) >> SCOUTFS_BLOCK_SM_SHIFT; + + while(iblock <= last) { + + ret = scoutfs_inode_index_lock_hold(inode, &ind_locks, false, + SIC_FALLOCATE_ONE()); + if (ret) + goto out; + + ret = fallocate_extents(sb, inode, iblock, last, lock); + + if (ret >= 0 && !(mode & FALLOC_FL_KEEP_SIZE)) { + end = (iblock + ret) << SCOUTFS_BLOCK_SM_SHIFT; + if (end > offset + len) + end = offset + len; + if (end > i_size_read(inode)) + i_size_write(inode, end); + } + if (ret >= 0) + scoutfs_update_inode_item(inode, lock, &ind_locks); + scoutfs_release_trans(sb); + scoutfs_inode_index_unlock(sb, &ind_locks); + + if (ret <= 0) + goto out; + + iblock += ret; + ret = 0; + } + +out: + scoutfs_unlock(sb, lock, SCOUTFS_LOCK_WRITE); + mutex_unlock(&inode->i_mutex); + + trace_scoutfs_data_fallocate(sb, ino, mode, offset, len, ret); + return ret; +} + +/* + * A special case of initializing a single large offline extent. This + * chooses not to deal with any existing extents. It can only be used + * on regular files with no data extents. It's used to restore a file + * with an offline extent which can then trigger staging. + * + * The caller has taken care of locking the inode. We're updating the + * inode offline count as we create the offline extent so we take care + * of the index locking, updating, and transaction. + */ +int scoutfs_data_init_offline_extent(struct inode *inode, u64 size, + struct scoutfs_lock *lock) + +{ + struct super_block *sb = inode->i_sb; + struct data_ext_args args = { + .ino = scoutfs_ino(inode), + .inode = inode, + .lock = lock, + }; + const u64 count = DIV_ROUND_UP(size, SCOUTFS_BLOCK_SM_SIZE); + LIST_HEAD(ind_locks); + u64 on; + u64 off; + int ret; + + scoutfs_inode_get_onoff(inode, &on, &off); + + /* caller should have checked */ + if (on > 0 || off > 0) { + ret = -EINVAL; + goto out; + } + + /* we're updating meta_seq with offline block count */ + ret = scoutfs_inode_index_lock_hold(inode, &ind_locks, false, + SIC_SETATTR_MORE()); + if (ret < 0) + goto out; + + ret = scoutfs_dirty_inode_item(inode, lock); + if (ret < 0) + goto unlock; + + ret = scoutfs_ext_insert(sb, &data_ext_ops, &args, + 0, count, 0, SEF_OFFLINE); + if (ret < 0) + goto unlock; + + scoutfs_update_inode_item(inode, lock, &ind_locks); + +unlock: + scoutfs_release_trans(sb); + scoutfs_inode_index_unlock(sb, &ind_locks); + ret = 0; +out: + return ret; +} + +/* + * This copies to userspace :/ + */ +static int fill_extent(struct fiemap_extent_info *fieinfo, + struct scoutfs_extent *ext, u32 fiemap_flags) +{ + u32 flags; + + if (ext->len == 0) + return 0; + + flags = fiemap_flags; + if (ext->flags & SEF_OFFLINE) + flags |= FIEMAP_EXTENT_UNKNOWN; + else if (ext->flags & SEF_UNWRITTEN) + flags |= FIEMAP_EXTENT_UNWRITTEN; + + return fiemap_fill_next_extent(fieinfo, + ext->start << SCOUTFS_BLOCK_SM_SHIFT, + ext->map << SCOUTFS_BLOCK_SM_SHIFT, + ext->len << SCOUTFS_BLOCK_SM_SHIFT, + flags); +} + +/* + * Return all the file's extents whose blocks overlap with the caller's + * byte region. We set _LAST on the last extent and _UNKNOWN on offline + * extents. + */ +int scoutfs_data_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, + u64 start, u64 len) +{ + struct super_block *sb = inode->i_sb; + const u64 ino = scoutfs_ino(inode); + struct scoutfs_lock *lock = NULL; + struct scoutfs_extent ext; + struct scoutfs_extent cur; + struct data_ext_args args; + u32 last_flags; + u64 iblock; + u64 last; + int ret; + + if (len == 0) { + ret = 0; + goto out; + } + + ret = fiemap_check_flags(fieinfo, FIEMAP_FLAG_SYNC); + if (ret) + goto out; + + /* XXX overkill? */ + mutex_lock(&inode->i_mutex); + + ret = scoutfs_lock_inode(sb, SCOUTFS_LOCK_READ, 0, inode, &lock); + if (ret) + goto unlock; + + args.ino = ino; + args.inode = inode; + args.lock = lock; + + /* use a dummy extent to track */ + memset(&cur, 0, sizeof(cur)); + last_flags = 0; + + iblock = start >> SCOUTFS_BLOCK_SM_SHIFT; + last = (start + len - 1) >> SCOUTFS_BLOCK_SM_SHIFT; + + while (iblock <= last) { + ret = scoutfs_ext_next(sb, &data_ext_ops, &args, + iblock, 1, &ext); + if (ret < 0) { + if (ret == -ENOENT) + ret = 0; + last_flags = FIEMAP_EXTENT_LAST; + break; + } + + trace_scoutfs_data_fiemap_extent(sb, ino, &ext); + + if (ext.start > last) { + /* not setting _LAST, it's for end of file */ + ret = 0; + break; + } + + if (scoutfs_ext_can_merge(&cur, &ext)) { + /* merged extents could be greater than input len */ + cur.len += ext.len; + } else { + ret = fill_extent(fieinfo, &cur, 0); + if (ret != 0) + goto unlock; + cur = ext; + } + + iblock = ext.start + ext.len; + } + + if (cur.len) + ret = fill_extent(fieinfo, &cur, last_flags); +unlock: + scoutfs_unlock(sb, lock, SCOUTFS_LOCK_READ); + mutex_unlock(&inode->i_mutex); + +out: + if (ret == 1) + ret = 0; + + trace_scoutfs_data_fiemap(sb, start, len, ret); + + return ret; +} + +/* + * Insert a new waiter. This supports multiple tasks waiting for the + * same ino and iblock by also comparing waiters by their addresses. + */ +static void insert_offline_waiting(struct rb_root *root, + struct scoutfs_data_wait *ins) +{ + struct rb_node **node = &root->rb_node; + struct rb_node *parent = NULL; + struct scoutfs_data_wait *dw; + int cmp; + + while (*node) { + parent = *node; + dw = rb_entry(*node, struct scoutfs_data_wait, node); + + cmp = scoutfs_cmp_u64s(ins->ino, dw->ino) ?: + scoutfs_cmp_u64s(ins->iblock, dw->iblock) ?: + scoutfs_cmp(ins, dw); + if (cmp < 0) + node = &(*node)->rb_left; + else + node = &(*node)->rb_right; + } + + rb_link_node(&ins->node, parent, node); + rb_insert_color(&ins->node, root); +} + +static struct scoutfs_data_wait *next_data_wait(struct rb_root *root, u64 ino, + u64 iblock) +{ + struct rb_node **node = &root->rb_node; + struct rb_node *parent = NULL; + struct scoutfs_data_wait *next = NULL; + struct scoutfs_data_wait *dw; + int cmp; + + while (*node) { + parent = *node; + dw = rb_entry(*node, struct scoutfs_data_wait, node); + + /* go left when ino/iblock are equal to get first task */ + cmp = scoutfs_cmp_u64s(ino, dw->ino) ?: + scoutfs_cmp_u64s(iblock, dw->iblock); + if (cmp <= 0) { + node = &(*node)->rb_left; + next = dw; + } else if (cmp > 0) { + node = &(*node)->rb_right; + } + } + + return next; +} + +static struct scoutfs_data_wait *dw_next(struct scoutfs_data_wait *dw) +{ + struct rb_node *node = rb_next(&dw->node); + if (node) + return container_of(node, struct scoutfs_data_wait, node); + return NULL; +} + +/* + * Check if we should wait by looking for extents whose flags match. + * Returns 0 if no extents were found or any error encountered. + * + * The caller must have locked the extents before calling, both across + * mounts and within this mount. + * + * Returns 1 if any file extents in the caller's region matched. If the + * wait struct is provided then it is initialized to be woken when the + * extents change after the caller unlocks after the check. The caller + * must come through _data_wait() to clean up the wait struct if we set + * it up. + */ +int scoutfs_data_wait_check(struct inode *inode, loff_t pos, loff_t len, + u8 sef, u8 op, struct scoutfs_data_wait *dw, + struct scoutfs_lock *lock) +{ + struct super_block *sb = inode->i_sb; + const u64 ino = scoutfs_ino(inode); + struct data_ext_args args = { + .ino = ino, + .inode = inode, + .lock = lock, + }; + DECLARE_DATA_WAIT_ROOT(sb, rt); + DECLARE_DATA_WAITQ(inode, wq); + struct scoutfs_extent ext = {0,}; + u64 iblock; + u64 last_block; + u64 on; + u64 off; + int ret = 0; + + if (WARN_ON_ONCE(sef & SEF_UNKNOWN) || + WARN_ON_ONCE(op & SCOUTFS_IOC_DWO_UNKNOWN) || + WARN_ON_ONCE(dw && !RB_EMPTY_NODE(&dw->node)) || + WARN_ON_ONCE(pos + len < pos)) { + ret = -EINVAL; + goto out; + } + + if ((sef & SEF_OFFLINE)) { + scoutfs_inode_get_onoff(inode, &on, &off); + if (off == 0) { + ret = 0; + goto out; + } + } + + iblock = pos >> SCOUTFS_BLOCK_SM_SHIFT; + last_block = (pos + len - 1) >> SCOUTFS_BLOCK_SM_SHIFT; + + while(iblock <= last_block) { + ret = scoutfs_ext_next(sb, &data_ext_ops, &args, + iblock, 1, &ext); + if (ret < 0) { + if (ret == -ENOENT) + ret = 0; + break; + } + + if (ext.start > last_block) { + ret = 0; + break; + } + + if (sef & ext.flags) { + if (dw) { + dw->chg = atomic64_read(&wq->changed); + dw->ino = ino; + dw->iblock = max(iblock, ext.start); + dw->op = op; + + spin_lock(&rt->lock); + insert_offline_waiting(&rt->root, dw); + spin_unlock(&rt->lock); + } + + ret = 1; + break; + } + + iblock = ext.start + ext.len; + } + +out: + trace_scoutfs_data_wait_check(sb, ino, pos, len, sef, op, &ext, ret); + + return ret; +} + +bool scoutfs_data_wait_found(struct scoutfs_data_wait *dw) +{ + return !RB_EMPTY_NODE(&dw->node); +} + +int scoutfs_data_wait_check_iov(struct inode *inode, const struct iovec *iov, + unsigned long nr_segs, loff_t pos, u8 sef, + u8 op, struct scoutfs_data_wait *dw, + struct scoutfs_lock *lock) +{ + unsigned long i; + int ret = 0; + + for (i = 0; i < nr_segs; i++) { + if (iov[i].iov_len == 0) + continue; + + ret = scoutfs_data_wait_check(inode, pos, iov[i].iov_len, sef, + op, dw, lock); + if (ret != 0) + break; + + pos += iov[i].iov_len; + } + + return ret; +} + +int scoutfs_data_wait(struct inode *inode, struct scoutfs_data_wait *dw) +{ + DECLARE_DATA_WAIT_ROOT(inode->i_sb, rt); + DECLARE_DATA_WAITQ(inode, wq); + int ret; + + ret = wait_event_interruptible(wq->waitq, + atomic64_read(&wq->changed) != dw->chg); + + spin_lock(&rt->lock); + rb_erase(&dw->node, &rt->root); + RB_CLEAR_NODE(&dw->node); + if (!ret && dw->err) + ret = dw->err; + spin_unlock(&rt->lock); + + return ret; +} + +void scoutfs_data_wait_changed(struct inode *inode) +{ + DECLARE_DATA_WAITQ(inode, wq); + + atomic64_inc(&wq->changed); + wake_up(&wq->waitq); +} + +long scoutfs_data_wait_err(struct inode *inode, u64 sblock, u64 eblock, + u64 op, long err) +{ + struct super_block *sb = inode->i_sb; + const u64 ino = scoutfs_ino(inode); + DECLARE_DATA_WAIT_ROOT(sb, rt); + struct scoutfs_data_wait *dw; + long nr = 0; + + if (!err) + return 0; + + spin_lock(&rt->lock); + + for (dw = next_data_wait(&rt->root, ino, sblock); + dw; dw = dw_next(dw)) { + if (dw->ino != ino || dw->iblock > eblock) + break; + if ((dw->op & op) && !dw->err) { + dw->err = err; + nr++; + } + } + + spin_unlock(&rt->lock); + if (nr) + scoutfs_data_wait_changed(inode); + return nr; +} + +int scoutfs_data_waiting(struct super_block *sb, u64 ino, u64 iblock, + struct scoutfs_ioctl_data_waiting_entry *dwe, + unsigned int nr) +{ + DECLARE_DATA_WAIT_ROOT(sb, rt); + struct scoutfs_data_wait *dw; + int ret = 0; + + spin_lock(&rt->lock); + + dw = next_data_wait(&rt->root, ino, iblock); + while (dw && ret < nr) { + + dwe->ino = dw->ino; + dwe->iblock = dw->iblock; + dwe->op = dw->op; + + while ((dw = dw_next(dw)) && + (dw->ino == dwe->ino && dw->iblock == dwe->iblock)) { + dwe->op |= dw->op; + } + + dwe++; + ret++; + } + + spin_unlock(&rt->lock); + + return ret; +} + +const struct address_space_operations scoutfs_file_aops = { + .readpage = scoutfs_readpage, + .readpages = scoutfs_readpages, + .writepage = scoutfs_writepage, + .writepages = scoutfs_writepages, + .write_begin = scoutfs_write_begin, + .write_end = scoutfs_write_end, +}; + +const struct file_operations scoutfs_file_fops = { + .read = do_sync_read, + .write = do_sync_write, + .aio_read = scoutfs_file_aio_read, + .aio_write = scoutfs_file_aio_write, + .unlocked_ioctl = scoutfs_ioctl, + .fsync = scoutfs_file_fsync, + .llseek = scoutfs_file_llseek, + .fallocate = scoutfs_fallocate, +}; + +void scoutfs_data_init_btrees(struct super_block *sb, + struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_log_trees *lt) +{ + DECLARE_DATA_INFO(sb, datinf); + + mutex_lock(&datinf->mutex); + + datinf->alloc = alloc; + datinf->wri = wri; + datinf->data_avail = lt->data_avail; + datinf->data_freed = lt->data_freed; + + mutex_unlock(&datinf->mutex); +} + +void scoutfs_data_get_btrees(struct super_block *sb, + struct scoutfs_log_trees *lt) +{ + DECLARE_DATA_INFO(sb, datinf); + + mutex_lock(&datinf->mutex); + + lt->data_avail = datinf->data_avail; + lt->data_freed = datinf->data_freed; + + mutex_unlock(&datinf->mutex); +} + +/* + * This should be called before preparing the allocators for the commit + * because it can allocate and free btree blocks in the data allocator. + */ +int scoutfs_data_prepare_commit(struct super_block *sb) +{ + DECLARE_DATA_INFO(sb, datinf); + int ret; + + mutex_lock(&datinf->mutex); + if (datinf->cached_ext.len) { + ret = scoutfs_free_data(sb, datinf->alloc, datinf->wri, + &datinf->data_avail, + datinf->cached_ext.start, + datinf->cached_ext.len); + if (ret == 0) + memset(&datinf->cached_ext, 0, + sizeof(datinf->cached_ext)); + } else { + ret = 0; + } + mutex_unlock(&datinf->mutex); + + return ret; +} + +/* + * This isn't serializing with allocators so it can be a bit racey. + */ +u64 scoutfs_data_alloc_free_bytes(struct super_block *sb) +{ + DECLARE_DATA_INFO(sb, datinf); + + return le64_to_cpu(datinf->data_avail.total_len) << + SCOUTFS_BLOCK_SM_SHIFT; +} + +int scoutfs_data_setup(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct data_info *datinf; + + datinf = kzalloc(sizeof(struct data_info), GFP_KERNEL); + if (!datinf) + return -ENOMEM; + + datinf->sb = sb; + mutex_init(&datinf->mutex); + + sbi->data_info = datinf; + return 0; +} + +void scoutfs_data_destroy(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct data_info *datinf = sbi->data_info; + + if (datinf) { + sbi->data_info = NULL; + kfree(datinf); + } +} diff --git a/kmod/src/data.h b/kmod/src/data.h new file mode 100644 index 00000000..09a64fe7 --- /dev/null +++ b/kmod/src/data.h @@ -0,0 +1,91 @@ +#ifndef _SCOUTFS_DATA_H_ +#define _SCOUTFS_DATA_H_ + +struct scoutfs_lock; +struct scoutfs_ioctl_data_waiting_entry; + +struct scoutfs_data_wait_root { + spinlock_t lock; + struct rb_root root; +}; + +#define DECLARE_DATA_WAIT_ROOT(sb, nm) \ + struct scoutfs_data_wait_root *nm = &SCOUTFS_SB(sb)->data_wait_root + +struct scoutfs_data_waitq { + atomic64_t changed; + wait_queue_head_t waitq; +}; + +#define DECLARE_DATA_WAITQ(in, nm) \ + struct scoutfs_data_waitq *nm = &SCOUTFS_I(in)->data_waitq + +/* + * Tasks can wait for data extents. + */ +struct scoutfs_data_wait { + struct rb_node node; + u64 chg; + u64 ino; + u64 iblock; + long err; + u8 op; +}; + +#define DECLARE_DATA_WAIT(nm) \ + struct scoutfs_data_wait nm = { \ + .node.__rb_parent_color = (unsigned long)(&nm.node), \ + .err = 0, \ + } + +struct scoutfs_traced_extent { + u64 iblock; + u64 count; + u64 blkno; + u8 flags; +}; + +extern const struct address_space_operations scoutfs_file_aops; +extern const struct file_operations scoutfs_file_fops; +struct scoutfs_alloc; +struct scoutfs_block_writer; + +int scoutfs_data_truncate_items(struct super_block *sb, struct inode *inode, + u64 ino, u64 iblock, u64 last, bool offline, + struct scoutfs_lock *lock); +int scoutfs_data_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, + u64 start, u64 len); +long scoutfs_fallocate(struct file *file, int mode, loff_t offset, loff_t len); +int scoutfs_data_init_offline_extent(struct inode *inode, u64 size, + struct scoutfs_lock *lock); + +int scoutfs_data_wait_check(struct inode *inode, loff_t pos, loff_t len, + u8 sef, u8 op, struct scoutfs_data_wait *ow, + struct scoutfs_lock *lock); +int scoutfs_data_wait_check_iov(struct inode *inode, const struct iovec *iov, + unsigned long nr_segs, loff_t pos, u8 sef, + u8 op, struct scoutfs_data_wait *ow, + struct scoutfs_lock *lock); +bool scoutfs_data_wait_found(struct scoutfs_data_wait *ow); +int scoutfs_data_wait(struct inode *inode, + struct scoutfs_data_wait *ow); +void scoutfs_data_wait_changed(struct inode *inode); +long scoutfs_data_wait_err(struct inode *inode, u64 sblock, u64 eblock, u64 op, + long err); +int scoutfs_data_waiting(struct super_block *sb, u64 ino, u64 iblock, + struct scoutfs_ioctl_data_waiting_entry *dwe, + unsigned int nr); + +void scoutfs_data_init_btrees(struct super_block *sb, + struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_log_trees *lt); +void scoutfs_data_get_btrees(struct super_block *sb, + struct scoutfs_log_trees *lt); +int scoutfs_data_prepare_commit(struct super_block *sb); +u64 scoutfs_data_alloc_free_bytes(struct super_block *sb); + +int scoutfs_data_setup(struct super_block *sb); +void scoutfs_data_destroy(struct super_block *sb); + +#endif diff --git a/kmod/src/dir.c b/kmod/src/dir.c new file mode 100644 index 00000000..8cbc20f0 --- /dev/null +++ b/kmod/src/dir.c @@ -0,0 +1,1805 @@ +/* + * Copyright (C) 2016 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include +#include +#include +#include + +#include "format.h" +#include "file.h" +#include "dir.h" +#include "inode.h" +#include "ioctl.h" +#include "key.h" +#include "msg.h" +#include "super.h" +#include "trans.h" +#include "xattr.h" +#include "item.h" +#include "lock.h" +#include "hash.h" +#include "counters.h" +#include "scoutfs_trace.h" + +/* + * Directory entries are stored in three different items. Each has the + * same key format and all have identical values which contain the full + * entry name. + * + * Entries for name lookup are stored at the hash of the name and the + * readdir position. Including the position lets us create names + * without having to read the items to check for hash collisions. + * Lookup iterates over all the positions with the same hash values and + * compares the names. + * + * Entries for readdir are stored in an increasing unique readdir + * position. This results in returning entries in creation order which + * matches inode allocation order and avoids random inode access + * patterns during readdir. + * + * Entries for link backref traversal are stored at the target inode + * sorted by the parent dir and the entry's position in the parent dir. + * This keeps link backref users away from the higher contention area of + * dirent items in parent dirs. + * + * All the entries have a dirent struct with the full name in their + * value. The dirent struct contains the name hash and readdir position + * so that any item use can reference all the items for a given entry. + * This is important for deleting all the items given a dentry that was + * populated by lookup. + */ + +static unsigned int mode_to_type(umode_t mode) +{ +#define S_SHIFT 12 + static unsigned char mode_types[S_IFMT >> S_SHIFT] = { + [S_IFIFO >> S_SHIFT] = SCOUTFS_DT_FIFO, + [S_IFCHR >> S_SHIFT] = SCOUTFS_DT_CHR, + [S_IFDIR >> S_SHIFT] = SCOUTFS_DT_DIR, + [S_IFBLK >> S_SHIFT] = SCOUTFS_DT_BLK, + [S_IFREG >> S_SHIFT] = SCOUTFS_DT_REG, + [S_IFLNK >> S_SHIFT] = SCOUTFS_DT_LNK, + [S_IFSOCK >> S_SHIFT] = SCOUTFS_DT_SOCK, + }; + + return mode_types[(mode & S_IFMT) >> S_SHIFT]; +#undef S_SHIFT +} + +static unsigned int dentry_type(enum scoutfs_dentry_type type) +{ + static unsigned char types[] = { + [SCOUTFS_DT_FIFO] = DT_FIFO, + [SCOUTFS_DT_CHR] = DT_CHR, + [SCOUTFS_DT_DIR] = DT_DIR, + [SCOUTFS_DT_BLK] = DT_BLK, + [SCOUTFS_DT_REG] = DT_REG, + [SCOUTFS_DT_LNK] = DT_LNK, + [SCOUTFS_DT_SOCK] = DT_SOCK, + [SCOUTFS_DT_WHT] = DT_WHT, + }; + + if (type < ARRAY_SIZE(types)) + return types[type]; + + return DT_UNKNOWN; +} + +/* + * @lock_cov: tells revalidation that the dentry is still locked and valid. + * + * @pos, @hash: lets us remove items on final unlink without having to + * look them up. + */ +struct dentry_info { + struct scoutfs_lock_coverage lock_cov; + u64 hash; + u64 pos; +}; + +static struct kmem_cache *dentry_info_cache; + +static void scoutfs_d_release(struct dentry *dentry) +{ + struct super_block *sb = dentry->d_sb; + struct dentry_info *di = dentry->d_fsdata; + + if (di) { + scoutfs_lock_del_coverage(sb, &di->lock_cov); + kmem_cache_free(dentry_info_cache, di); + dentry->d_fsdata = NULL; + } +} + +static int scoutfs_d_revalidate(struct dentry *dentry, unsigned int flags); + +static const struct dentry_operations scoutfs_dentry_ops = { + .d_release = scoutfs_d_release, + .d_revalidate = scoutfs_d_revalidate, +}; + +static int alloc_dentry_info(struct dentry *dentry) +{ + struct dentry_info *di; + + /* XXX read mb? */ + if (dentry->d_fsdata) + return 0; + + di = kmem_cache_zalloc(dentry_info_cache, GFP_NOFS); + if (!di) + return -ENOMEM; + + scoutfs_lock_init_coverage(&di->lock_cov); + + spin_lock(&dentry->d_lock); + if (!dentry->d_fsdata) { + dentry->d_fsdata = di; + d_set_d_op(dentry, &scoutfs_dentry_ops); + } + spin_unlock(&dentry->d_lock); + + if (di != dentry->d_fsdata) + kmem_cache_free(dentry_info_cache, di); + + return 0; +} + +static void update_dentry_info(struct super_block *sb, struct dentry *dentry, + u64 hash, u64 pos, struct scoutfs_lock *lock) +{ + struct dentry_info *di = dentry->d_fsdata; + + if (WARN_ON_ONCE(di == NULL)) + return; + + scoutfs_lock_add_coverage(sb, lock, &di->lock_cov); + di->hash = hash; + di->pos = pos; +} + +static u64 dentry_info_hash(struct dentry *dentry) +{ + struct dentry_info *di = dentry->d_fsdata; + + if (WARN_ON_ONCE(di == NULL)) + return 0; + + return di->hash; +} + +static u64 dentry_info_pos(struct dentry *dentry) +{ + struct dentry_info *di = dentry->d_fsdata; + + if (WARN_ON_ONCE(di == NULL)) + return 0; + + return di->pos; +} + +static void init_dirent_key(struct scoutfs_key *key, u8 type, u64 ino, + u64 major, u64 minor) +{ + *key = (struct scoutfs_key) { + .sk_zone = SCOUTFS_FS_ZONE, + .skd_ino = cpu_to_le64(ino), + .sk_type = type, + .skd_major = cpu_to_le64(major), + .skd_minor = cpu_to_le64(minor), + }; +} + +static unsigned int dirent_bytes(unsigned int name_len) +{ + return offsetof(struct scoutfs_dirent, name[name_len]); +} + +static struct scoutfs_dirent *alloc_dirent(unsigned int name_len) +{ + return kmalloc(dirent_bytes(name_len), GFP_NOFS); +} + +/* + * Test a bit number as though an array of bytes is a large len-bit + * big-endian value. nr 0 is the LSB of the final byte, nr (len - 1) is + * the MSB of the first byte. + */ +static int test_be_bytes_bit(int nr, const char *bytes, int len) +{ + return bytes[(len - 1 - nr) >> 3] & (1 << (nr & 7)); +} + +/* + * Generate a 32bit "fingerprint" of the name by extracting 32 evenly + * distributed bits from the name. The intent is to have the sort order + * of the fingerprints reflect the memcmp() sort order of the names + * while mapping large names down to small fs keys. + * + * Names that are smaller than 32bits are biased towards the high bits + * of the fingerprint so that most significant bits of the fingerprints + * consistently reflect the initial characters of the names. + */ +static u32 dirent_name_fingerprint(const char *name, unsigned int name_len) +{ + int name_bits = name_len * 8; + int skip = max(name_bits / 32, 1); + u32 fp = 0; + int f; + int n; + + for (f = 31, n = name_bits - 1; f >= 0 && n >= 0; f--, n -= skip) + fp |= !!test_be_bytes_bit(n, name, name_bits) << f; + + return fp; +} + +static u64 dirent_name_hash(const char *name, unsigned int name_len) +{ + return scoutfs_hash32(name, name_len) | + ((u64)dirent_name_fingerprint(name, name_len) << 32); +} + +static u64 dirent_names_equal(const char *a_name, unsigned int a_len, + const char *b_name, unsigned int b_len) +{ + return a_len == b_len && memcmp(a_name, b_name, a_len) == 0; +} + +/* + * Looks for the dirent item and fills the caller's dirent if it finds + * it. Returns item lookup errors including -ENOENT if it's not found. + */ +static int lookup_dirent(struct super_block *sb, u64 dir_ino, const char *name, + unsigned name_len, u64 hash, + struct scoutfs_dirent *dent_ret, + struct scoutfs_lock *lock) +{ + struct scoutfs_key last_key; + struct scoutfs_key key; + struct scoutfs_dirent *dent = NULL; + int ret; + + dent = alloc_dirent(SCOUTFS_NAME_LEN); + if (!dent) { + ret = -ENOMEM; + goto out; + } + + init_dirent_key(&key, SCOUTFS_DIRENT_TYPE, dir_ino, hash, 0); + init_dirent_key(&last_key, SCOUTFS_DIRENT_TYPE, dir_ino, hash, U64_MAX); + + for (;;) { + ret = scoutfs_item_next(sb, &key, &last_key, dent, + dirent_bytes(SCOUTFS_NAME_LEN), lock); + if (ret < 0) + break; + + ret -= sizeof(struct scoutfs_dirent); + if (ret < 1 || ret > SCOUTFS_NAME_LEN) { + scoutfs_corruption(sb, SC_DIRENT_NAME_LEN, + corrupt_dirent_name_len, + "dir_ino %llu hash %llu key "SK_FMT" len %d", + dir_ino, hash, SK_ARG(&key), ret); + ret = -EIO; + goto out; + } + + if (dirent_names_equal(name, name_len, dent->name, ret)) { + *dent_ret = *dent; + ret = 0; + break; + } + + if (le64_to_cpu(key.skd_minor) == U64_MAX) { + ret = -ENOENT; + break; + } + le64_add_cpu(&key.skd_minor, 1); + } + +out: + kfree(dent); + return ret; +} + +static int scoutfs_d_revalidate(struct dentry *dentry, unsigned int flags) +{ + struct super_block *sb = dentry->d_sb; + struct dentry_info *di = dentry->d_fsdata; + struct dentry *parent = dget_parent(dentry); + struct scoutfs_lock *lock = NULL; + struct scoutfs_dirent dent; + bool is_covered = false; + struct inode *dir; + u64 dentry_ino; + int ret; + + /* don't think this happens but we can find out */ + if (IS_ROOT(dentry)) { + scoutfs_inc_counter(sb, dentry_revalidate_root); + if (!dentry->d_inode || + (scoutfs_ino(dentry->d_inode) != SCOUTFS_ROOT_INO)) { + ret = -EIO; + } else { + ret = 1; + } + goto out; + } + + /* XXX what are the rules for _RCU? */ + if (flags & LOOKUP_RCU) { + scoutfs_inc_counter(sb, dentry_revalidate_rcu); + ret = -ECHILD; + goto out; + } + + if (WARN_ON_ONCE(di == NULL)) { + ret = 0; + goto out; + } + + is_covered = scoutfs_lock_is_covered(sb, &di->lock_cov); + if (is_covered) { + scoutfs_inc_counter(sb, dentry_revalidate_locked); + ret = 1; + goto out; + } + + if (!parent || !parent->d_inode) { + scoutfs_inc_counter(sb, dentry_revalidate_orphan); + ret = 0; + goto out; + } + dir = parent->d_inode; + + ret = scoutfs_lock_inode(sb, SCOUTFS_LOCK_READ, 0, dir, &lock); + if (ret) + goto out; + + ret = lookup_dirent(sb, scoutfs_ino(dir), + dentry->d_name.name, dentry->d_name.len, + dirent_name_hash(dentry->d_name.name, + dentry->d_name.len), + &dent, lock); + if (ret == -ENOENT) { + dent.ino = 0; + dent.hash = 0; + dent.pos = 0; + } else if (ret < 0) { + goto out; + } + + dentry_ino = dentry->d_inode ? scoutfs_ino(dentry->d_inode) : 0; + + if ((dentry_ino == le64_to_cpu(dent.ino))) { + update_dentry_info(sb, dentry, le64_to_cpu(dent.hash), + le64_to_cpu(dent.pos), lock); + scoutfs_inc_counter(sb, dentry_revalidate_valid); + ret = 1; + } else { + scoutfs_inc_counter(sb, dentry_revalidate_invalid); + ret = 0; + } + +out: + trace_scoutfs_d_revalidate(sb, dentry, flags, parent, is_covered, ret); + + dput(parent); + scoutfs_unlock(sb, lock, SCOUTFS_LOCK_READ); + + if (ret < 0 && ret != -ECHILD) + scoutfs_inc_counter(sb, dentry_revalidate_error); + + return ret; +} + +/* + * Because of rename, locks are ordered by inode number. To hold the + * dir lock while calling iget, we might have to already hold a lesser + * inode's lock while telling iget whether or not to lock. Instead of + * adding all those moving pieces we drop the dir lock before calling + * iget. We don't reuse inode numbers so we don't have to worry about + * the target of the link changing. We will only follow the entry as it + * existed before or after whatever modification is happening under the + * dir lock and that can already legally race before or after our + * lookup. + */ +static struct dentry *scoutfs_lookup(struct inode *dir, struct dentry *dentry, + unsigned int flags) +{ + struct super_block *sb = dir->i_sb; + struct scoutfs_lock *dir_lock = NULL; + struct scoutfs_dirent dent; + struct inode *inode; + u64 ino = 0; + u64 hash; + int ret; + + hash = dirent_name_hash(dentry->d_name.name, dentry->d_name.len); + + if (dentry->d_name.len > SCOUTFS_NAME_LEN) { + ret = -ENAMETOOLONG; + goto out; + } + + ret = alloc_dentry_info(dentry); + if (ret) + goto out; + + ret = scoutfs_lock_inode(sb, SCOUTFS_LOCK_READ, 0, dir, &dir_lock); + if (ret) + goto out; + + ret = lookup_dirent(sb, scoutfs_ino(dir), dentry->d_name.name, + dentry->d_name.len, hash, &dent, dir_lock); + if (ret == -ENOENT) { + ino = 0; + ret = 0; + } else if (ret == 0) { + ino = le64_to_cpu(dent.ino); + update_dentry_info(sb, dentry, le64_to_cpu(dent.hash), + le64_to_cpu(dent.pos), dir_lock); + } + scoutfs_unlock(sb, dir_lock, SCOUTFS_LOCK_READ); + +out: + if (ret < 0) + inode = ERR_PTR(ret); + else if (ino == 0) + inode = NULL; + else + inode = scoutfs_iget(sb, ino); + + return d_splice_alias(inode, dentry); +} + +/* + * readdir simply iterates over the dirent items for the dir inode and + * uses their offset as the readdir position. + * + * It will need to be careful not to read past the region of the dirent + * hash offset keys that it has access to. + */ +static int KC_DECLARE_READDIR(scoutfs_readdir, struct file *file, + void *dirent, kc_readdir_ctx_t ctx) +{ + struct inode *inode = file_inode(file); + struct super_block *sb = inode->i_sb; + struct scoutfs_dirent *dent; + struct scoutfs_key key; + struct scoutfs_key last_key; + struct scoutfs_lock *dir_lock; + int name_len; + u64 pos; + int ret; + + if (!kc_dir_emit_dots(file, dirent, ctx)) + return 0; + + dent = alloc_dirent(SCOUTFS_NAME_LEN); + if (!dent) { + ret = -ENOMEM; + goto out; + } + + init_dirent_key(&last_key, SCOUTFS_READDIR_TYPE, scoutfs_ino(inode), + SCOUTFS_DIRENT_LAST_POS, 0); + + ret = scoutfs_lock_inode(sb, SCOUTFS_LOCK_READ, 0, inode, &dir_lock); + if (ret) + goto out; + + for (;;) { + init_dirent_key(&key, SCOUTFS_READDIR_TYPE, scoutfs_ino(inode), + kc_readdir_pos(file, ctx), 0); + + ret = scoutfs_item_next(sb, &key, &last_key, dent, + dirent_bytes(SCOUTFS_NAME_LEN), + dir_lock); + if (ret < 0) { + if (ret == -ENOENT) + ret = 0; + break; + } + + name_len = ret - sizeof(struct scoutfs_dirent); + if (name_len < 1 || name_len > SCOUTFS_NAME_LEN) { + scoutfs_corruption(sb, SC_DIRENT_READDIR_NAME_LEN, + corrupt_dirent_readdir_name_len, + "dir_ino %llu pos %llu key "SK_FMT" len %d", + scoutfs_ino(inode), + kc_readdir_pos(file, ctx), + SK_ARG(&key), name_len); + ret = -EIO; + goto out; + } + + pos = le64_to_cpu(key.skd_major); + kc_readdir_pos(file, ctx) = pos; + + if (!kc_dir_emit(ctx, dirent, dent->name, name_len, pos, + le64_to_cpu(dent->ino), + dentry_type(dent->type))) { + ret = 0; + break; + } + + kc_readdir_pos(file, ctx) = pos + 1; + } + +out: + scoutfs_unlock(sb, dir_lock, SCOUTFS_LOCK_READ); + + kfree(dent); + return ret; +} + +/* + * Add all the items for the named link to the inode in the dir. Only + * items are modified. The caller is responsible for locking, entering + * a transaction, dirtying items, and managing the vfs structs. + * + * If this returns an error then nothing will have changed. + */ +static int add_entry_items(struct super_block *sb, u64 dir_ino, u64 hash, + u64 pos, const char *name, unsigned name_len, + u64 ino, umode_t mode, struct scoutfs_lock *dir_lock, + struct scoutfs_lock *inode_lock) +{ + struct scoutfs_key rdir_key; + struct scoutfs_key ent_key; + struct scoutfs_key lb_key; + struct scoutfs_dirent *dent; + bool del_ent = false; + bool del_rdir = false; + int ret; + + dent = alloc_dirent(name_len); + if (!dent) { + ret = -ENOMEM; + goto out; + } + + /* initialize the dent */ + dent->ino = cpu_to_le64(ino); + dent->hash = cpu_to_le64(hash); + dent->pos = cpu_to_le64(pos); + dent->type = mode_to_type(mode); + memcpy(dent->name, name, name_len); + + init_dirent_key(&ent_key, SCOUTFS_DIRENT_TYPE, dir_ino, hash, pos); + init_dirent_key(&rdir_key, SCOUTFS_READDIR_TYPE, dir_ino, pos, 0); + init_dirent_key(&lb_key, SCOUTFS_LINK_BACKREF_TYPE, ino, dir_ino, pos); + + ret = scoutfs_item_create(sb, &ent_key, dent, dirent_bytes(name_len), + dir_lock); + if (ret) + goto out; + del_ent = true; + + ret = scoutfs_item_create(sb, &rdir_key, dent, dirent_bytes(name_len), + dir_lock); + if (ret) + goto out; + del_rdir = true; + + ret = scoutfs_item_create(sb, &lb_key, dent, dirent_bytes(name_len), + inode_lock); +out: + if (ret < 0) { + if (del_ent) + scoutfs_item_delete(sb, &ent_key, dir_lock); + if (del_rdir) + scoutfs_item_delete(sb, &rdir_key, dir_lock); + } + + kfree(dent); + + return ret; +} + +/* + * Delete all the items for the named link to the inode in the dir. + * Only items are modified. The caller is responsible for locking, + * entering a transaction, dirtying items, and managing the vfs structs. + * + * If this returns an error then nothing will have changed. + */ +static int del_entry_items(struct super_block *sb, u64 dir_ino, u64 hash, + u64 pos, u64 ino, struct scoutfs_lock *dir_lock, + struct scoutfs_lock *inode_lock) +{ + struct scoutfs_key rdir_key; + struct scoutfs_key ent_key; + struct scoutfs_key lb_key; + int ret; + + init_dirent_key(&ent_key, SCOUTFS_DIRENT_TYPE, dir_ino, hash, pos); + init_dirent_key(&rdir_key, SCOUTFS_READDIR_TYPE, dir_ino, pos, 0); + init_dirent_key(&lb_key, SCOUTFS_LINK_BACKREF_TYPE, ino, dir_ino, pos); + + ret = scoutfs_item_dirty(sb, &ent_key, dir_lock) ?: + scoutfs_item_dirty(sb, &rdir_key, dir_lock) ?: + scoutfs_item_dirty(sb, &lb_key, inode_lock); + if (ret == 0) { + ret = scoutfs_item_delete(sb, &ent_key, dir_lock) ?: + scoutfs_item_delete(sb, &rdir_key, dir_lock) ?: + scoutfs_item_delete(sb, &lb_key, inode_lock); + BUG_ON(ret); /* _dirty should have guaranteed success */ + } + + return ret; +} + +/* + * Inode creation needs to hold dir and inode locks which can be greater + * or less than each other. It seems easiest to keep the dual locking + * here like it is for all the other dual locking of established inodes. + * Except we don't have the inode struct yet when we're getting locks, + * so we roll our own comparion between the two instead of pushing + * complexity down the locking paths that acquire existing inodes in + * order. + */ +static struct inode *lock_hold_create(struct inode *dir, struct dentry *dentry, + umode_t mode, dev_t rdev, + const struct scoutfs_item_count cnt, + struct scoutfs_lock **dir_lock, + struct scoutfs_lock **inode_lock, + struct list_head *ind_locks) +{ + struct super_block *sb = dir->i_sb; + struct inode *inode; + u64 ind_seq; + int ret = 0; + u64 ino; + + ret = alloc_dentry_info(dentry); + if (ret) + return ERR_PTR(ret); + + ret = scoutfs_alloc_ino(sb, S_ISDIR(mode), &ino); + if (ret) + return ERR_PTR(ret); + + if (ino < scoutfs_ino(dir)) { + ret = scoutfs_lock_ino(sb, SCOUTFS_LOCK_WRITE, 0, ino, + inode_lock) ?: + scoutfs_lock_inode(sb, SCOUTFS_LOCK_WRITE, + SCOUTFS_LKF_REFRESH_INODE, dir, + dir_lock); + } else { + ret = scoutfs_lock_inode(sb, SCOUTFS_LOCK_WRITE, + SCOUTFS_LKF_REFRESH_INODE, dir, + dir_lock) ?: + scoutfs_lock_ino(sb, SCOUTFS_LOCK_WRITE, 0, ino, + inode_lock); + } + if (ret) + goto out_unlock; + +retry: + ret = scoutfs_inode_index_start(sb, &ind_seq) ?: + scoutfs_inode_index_prepare(sb, ind_locks, dir, true) ?: + scoutfs_inode_index_prepare_ino(sb, ind_locks, ino, mode) ?: + scoutfs_inode_index_try_lock_hold(sb, ind_locks, ind_seq, cnt); + if (ret > 0) + goto retry; + if (ret) + goto out_unlock; + + inode = scoutfs_new_inode(sb, dir, mode, rdev, ino, *inode_lock); + if (IS_ERR(inode)) { + ret = PTR_ERR(inode); + goto out; + } + + ret = scoutfs_dirty_inode_item(dir, *dir_lock); +out: + if (ret) + scoutfs_release_trans(sb); +out_unlock: + if (ret) { + scoutfs_inode_index_unlock(sb, ind_locks); + scoutfs_unlock(sb, *dir_lock, SCOUTFS_LOCK_WRITE); + scoutfs_unlock(sb, *inode_lock, SCOUTFS_LOCK_WRITE); + *dir_lock = NULL; + *inode_lock = NULL; + + inode = ERR_PTR(ret); + } + + return inode; +} + +static int scoutfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, + dev_t rdev) +{ + struct super_block *sb = dir->i_sb; + struct inode *inode = NULL; + struct scoutfs_lock *dir_lock = NULL; + struct scoutfs_lock *inode_lock = NULL; + LIST_HEAD(ind_locks); + u64 hash; + u64 pos; + int ret; + + if (dentry->d_name.len > SCOUTFS_NAME_LEN) + return -ENAMETOOLONG; + + hash = dirent_name_hash(dentry->d_name.name, dentry->d_name.len); + inode = lock_hold_create(dir, dentry, mode, rdev, + SIC_MKNOD(dentry->d_name.len), + &dir_lock, &inode_lock, &ind_locks); + if (IS_ERR(inode)) + return PTR_ERR(inode); + + pos = SCOUTFS_I(dir)->next_readdir_pos++; + + ret = add_entry_items(sb, scoutfs_ino(dir), hash, pos, + dentry->d_name.name, dentry->d_name.len, + scoutfs_ino(inode), inode->i_mode, dir_lock, + inode_lock); + if (ret) + goto out; + + update_dentry_info(sb, dentry, hash, pos, dir_lock); + + i_size_write(dir, i_size_read(dir) + dentry->d_name.len); + dir->i_mtime = dir->i_ctime = CURRENT_TIME; + inode->i_mtime = inode->i_atime = inode->i_ctime = dir->i_mtime; + + if (S_ISDIR(mode)) { + inc_nlink(inode); + inc_nlink(dir); + } + + scoutfs_update_inode_item(inode, inode_lock, &ind_locks); + scoutfs_update_inode_item(dir, dir_lock, &ind_locks); + scoutfs_inode_index_unlock(sb, &ind_locks); + + insert_inode_hash(inode); + d_instantiate(dentry, inode); +out: + scoutfs_release_trans(sb); + scoutfs_inode_index_unlock(sb, &ind_locks); + scoutfs_unlock(sb, dir_lock, SCOUTFS_LOCK_WRITE); + scoutfs_unlock(sb, inode_lock, SCOUTFS_LOCK_WRITE); + + /* XXX delete the inode item here */ + if (ret && !IS_ERR_OR_NULL(inode)) + iput(inode); + return ret; +} + +/* XXX hmm, do something with excl? */ +static int scoutfs_create(struct inode *dir, struct dentry *dentry, + umode_t mode, bool excl) +{ + return scoutfs_mknod(dir, dentry, mode | S_IFREG, 0); +} + +static int scoutfs_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode) +{ + return scoutfs_mknod(dir, dentry, mode | S_IFDIR, 0); +} + +static int scoutfs_link(struct dentry *old_dentry, + struct inode *dir, struct dentry *dentry) +{ + struct inode *inode = old_dentry->d_inode; + struct super_block *sb = dir->i_sb; + struct scoutfs_lock *dir_lock; + struct scoutfs_lock *inode_lock = NULL; + LIST_HEAD(ind_locks); + u64 dir_size; + u64 ind_seq; + u64 hash; + u64 pos; + int ret; + + hash = dirent_name_hash(dentry->d_name.name, dentry->d_name.len); + + if (dentry->d_name.len > SCOUTFS_NAME_LEN) + return -ENAMETOOLONG; + + ret = scoutfs_lock_inodes(sb, SCOUTFS_LOCK_WRITE, + SCOUTFS_LKF_REFRESH_INODE, + dir, &dir_lock, inode, &inode_lock, + NULL, NULL, NULL, NULL); + if (ret) + return ret; + + if (inode->i_nlink >= SCOUTFS_LINK_MAX) { + ret = -EMLINK; + goto out_unlock; + } + + ret = alloc_dentry_info(dentry); + if (ret) + goto out_unlock; + + dir_size = i_size_read(dir) + dentry->d_name.len; +retry: + ret = scoutfs_inode_index_start(sb, &ind_seq) ?: + scoutfs_inode_index_prepare(sb, &ind_locks, dir, false) ?: + scoutfs_inode_index_prepare(sb, &ind_locks, inode, false) ?: + scoutfs_inode_index_try_lock_hold(sb, &ind_locks, ind_seq, + SIC_LINK(dentry->d_name.len)); + if (ret > 0) + goto retry; + if (ret) + goto out_unlock; + + ret = scoutfs_dirty_inode_item(dir, dir_lock); + if (ret) + goto out; + + pos = SCOUTFS_I(dir)->next_readdir_pos++; + + ret = add_entry_items(sb, scoutfs_ino(dir), hash, pos, + dentry->d_name.name, dentry->d_name.len, + scoutfs_ino(inode), inode->i_mode, dir_lock, + inode_lock); + if (ret) + goto out; + update_dentry_info(sb, dentry, hash, pos, dir_lock); + + i_size_write(dir, dir_size); + dir->i_mtime = dir->i_ctime = CURRENT_TIME; + inode->i_ctime = dir->i_mtime; + inc_nlink(inode); + + scoutfs_update_inode_item(inode, inode_lock, &ind_locks); + scoutfs_update_inode_item(dir, dir_lock, &ind_locks); + + atomic_inc(&inode->i_count); + d_instantiate(dentry, inode); +out: + scoutfs_release_trans(sb); +out_unlock: + scoutfs_inode_index_unlock(sb, &ind_locks); + scoutfs_unlock(sb, dir_lock, SCOUTFS_LOCK_WRITE); + scoutfs_unlock(sb, inode_lock, SCOUTFS_LOCK_WRITE); + return ret; +} + +static bool should_orphan(struct inode *inode) +{ + if (inode == NULL) + return false; + + if (S_ISDIR(inode->i_mode)) + return inode->i_nlink == 2; + + return inode->i_nlink == 1; +} + +/* + * Unlink removes the entry from its item and removes the item if ours + * was the only remaining entry. + */ +static int scoutfs_unlink(struct inode *dir, struct dentry *dentry) +{ + struct super_block *sb = dir->i_sb; + struct inode *inode = dentry->d_inode; + struct timespec ts = current_kernel_time(); + struct scoutfs_lock *inode_lock = NULL; + struct scoutfs_lock *dir_lock = NULL; + LIST_HEAD(ind_locks); + u64 ind_seq; + int ret = 0; + + ret = scoutfs_lock_inodes(sb, SCOUTFS_LOCK_WRITE, + SCOUTFS_LKF_REFRESH_INODE, + dir, &dir_lock, inode, &inode_lock, + NULL, NULL, NULL, NULL); + if (ret) + return ret; + + if (S_ISDIR(inode->i_mode) && i_size_read(inode)) { + ret = -ENOTEMPTY; + goto unlock; + } + +retry: + ret = scoutfs_inode_index_start(sb, &ind_seq) ?: + scoutfs_inode_index_prepare(sb, &ind_locks, dir, false) ?: + scoutfs_inode_index_prepare(sb, &ind_locks, inode, false) ?: + scoutfs_inode_index_try_lock_hold(sb, &ind_locks, ind_seq, + SIC_UNLINK(dentry->d_name.len)); + if (ret > 0) + goto retry; + if (ret) + goto unlock; + + ret = del_entry_items(sb, scoutfs_ino(dir), dentry_info_hash(dentry), + dentry_info_pos(dentry), scoutfs_ino(inode), + dir_lock, inode_lock); + if (ret) + goto out; + + if (should_orphan(inode)) { + /* + * Insert the orphan item before we modify any inode + * metadata so we can gracefully exit should it + * fail. + */ + ret = scoutfs_orphan_inode(inode); + WARN_ON_ONCE(ret); /* XXX returning error but items deleted */ + if (ret) + goto out; + } + + dir->i_ctime = ts; + dir->i_mtime = ts; + i_size_write(dir, i_size_read(dir) - dentry->d_name.len); + + inode->i_ctime = ts; + drop_nlink(inode); + if (S_ISDIR(inode->i_mode)) { + drop_nlink(dir); + drop_nlink(inode); + } + scoutfs_update_inode_item(inode, inode_lock, &ind_locks); + scoutfs_update_inode_item(dir, dir_lock, &ind_locks); + +out: + scoutfs_release_trans(sb); +unlock: + scoutfs_inode_index_unlock(sb, &ind_locks); + scoutfs_unlock(sb, dir_lock, SCOUTFS_LOCK_WRITE); + scoutfs_unlock(sb, inode_lock, SCOUTFS_LOCK_WRITE); + + return ret; +} + +static void init_symlink_key(struct scoutfs_key *key, u64 ino, u8 nr) +{ + *key = (struct scoutfs_key) { + .sk_zone = SCOUTFS_FS_ZONE, + .sks_ino = cpu_to_le64(ino), + .sk_type = SCOUTFS_SYMLINK_TYPE, + .sks_nr = cpu_to_le64(nr), + }; +} + +/* + * Operate on all the items that make up a symlink whose target might + * have to be split up into multiple items each with a maximally sized + * value. + * + * returns 0 or -errno from the item calls, particularly including + * EEXIST, EIO, or ENOENT if the item population doesn't match what was + * expected given the op. + * + * The target name can be null for deletion when val isn't used. Size + * still has to be provided to determine the number of items. + */ +enum symlink_ops { + SYM_CREATE = 0, + SYM_LOOKUP, + SYM_DELETE, +}; +static int symlink_item_ops(struct super_block *sb, enum symlink_ops op, u64 ino, + struct scoutfs_lock *lock, const char *target, + size_t size) +{ + struct scoutfs_key key; + unsigned bytes; + unsigned nr; + int ret; + int i; + + if (WARN_ON_ONCE(size == 0 || size > SCOUTFS_SYMLINK_MAX_SIZE || + op > SYM_DELETE)) + return -EINVAL; + + nr = DIV_ROUND_UP(size, SCOUTFS_MAX_VAL_SIZE); + for (i = 0; i < nr; i++) { + + init_symlink_key(&key, ino, i); + bytes = min_t(u64, size, SCOUTFS_MAX_VAL_SIZE); + + if (op == SYM_CREATE) + ret = scoutfs_item_create(sb, &key, (void *)target, + bytes, lock); + else if (op == SYM_LOOKUP) + ret = scoutfs_item_lookup_exact(sb, &key, + (void *)target, bytes, + lock); + else if (op == SYM_DELETE) + ret = scoutfs_item_delete(sb, &key, lock); + if (ret) + break; + + target += SCOUTFS_MAX_VAL_SIZE; + size -= bytes; + } + + return ret; +} + +/* + * Full a buffer with the null terminated symlink, point nd at it, and + * return it so put_link can free it once the vfs is done. + * + * We chose to pay the runtime cost of per-call allocation and copy + * overhead instead of wiring up symlinks to the page cache, storing + * each small link in a full page, and later having to reclaim them. + */ +static void *scoutfs_follow_link(struct dentry *dentry, struct nameidata *nd) +{ + struct inode *inode = dentry->d_inode; + struct super_block *sb = inode->i_sb; + struct scoutfs_lock *inode_lock = NULL; + char *path = NULL; + loff_t size; + int ret; + + ret = scoutfs_lock_inode(sb, SCOUTFS_LOCK_READ, + SCOUTFS_LKF_REFRESH_INODE, inode, &inode_lock); + if (ret) + return ERR_PTR(ret); + + size = i_size_read(inode); + + if (size == 0 || size > SCOUTFS_SYMLINK_MAX_SIZE) { + scoutfs_corruption(sb, SC_SYMLINK_INODE_SIZE, + corrupt_symlink_inode_size, + "ino %llu size %llu", + scoutfs_ino(inode), (u64)size); + ret = -EIO; + goto out; + } + + /* unlikely, but possible I suppose */ + if (size > PATH_MAX) { + ret = -ENAMETOOLONG; + goto out; + } + + path = kmalloc(size, GFP_NOFS); + if (!path) { + ret = -ENOMEM; + goto out; + } + + ret = symlink_item_ops(sb, SYM_LOOKUP, scoutfs_ino(inode), inode_lock, + path, size); + + if (ret == -ENOENT) { + scoutfs_corruption(sb, SC_SYMLINK_MISSING_ITEM, + corrupt_symlink_missing_item, + "ino %llu size %llu", scoutfs_ino(inode), + size); + ret = -EIO; + + } else if (ret == 0 && path[size - 1]) { + scoutfs_corruption(sb, SC_SYMLINK_NOT_NULL_TERM, + corrupt_symlink_not_null_term, + "ino %llu last %u", + scoutfs_ino(inode), path[size - 1]); + ret = -EIO; + } + +out: + if (ret < 0) { + kfree(path); + path = ERR_PTR(ret); + } else { + nd_set_link(nd, path); + } + scoutfs_unlock(sb, inode_lock, SCOUTFS_LOCK_READ); + return path; +} + +static void scoutfs_put_link(struct dentry *dentry, struct nameidata *nd, + void *cookie) +{ + if (!IS_ERR_OR_NULL(cookie)) + kfree(cookie); +} + +const struct inode_operations scoutfs_symlink_iops = { + .readlink = generic_readlink, + .follow_link = scoutfs_follow_link, + .put_link = scoutfs_put_link, + .getattr = scoutfs_getattr, + .setattr = scoutfs_setattr, + .setxattr = scoutfs_setxattr, + .getxattr = scoutfs_getxattr, + .listxattr = scoutfs_listxattr, + .removexattr = scoutfs_removexattr, +}; + +/* + * Symlink target paths can be annoyingly large. We store relatively + * rare large paths in multiple items. + */ +static int scoutfs_symlink(struct inode *dir, struct dentry *dentry, + const char *symname) +{ + struct super_block *sb = dir->i_sb; + const int name_len = strlen(symname) + 1; + struct inode *inode = NULL; + struct scoutfs_lock *dir_lock = NULL; + struct scoutfs_lock *inode_lock = NULL; + LIST_HEAD(ind_locks); + u64 hash; + u64 pos; + int ret; + + hash = dirent_name_hash(dentry->d_name.name, dentry->d_name.len); + + /* path_max includes null as does our value for nd_set_link */ + if (dentry->d_name.len > SCOUTFS_NAME_LEN || + name_len > PATH_MAX || name_len > SCOUTFS_SYMLINK_MAX_SIZE) + return -ENAMETOOLONG; + + ret = alloc_dentry_info(dentry); + if (ret) + return ret; + + inode = lock_hold_create(dir, dentry, S_IFLNK|S_IRWXUGO, 0, + SIC_SYMLINK(dentry->d_name.len, name_len), + &dir_lock, &inode_lock, &ind_locks); + if (IS_ERR(inode)) + return PTR_ERR(inode); + + ret = symlink_item_ops(sb, SYM_CREATE, scoutfs_ino(inode), inode_lock, + symname, name_len); + if (ret) + goto out; + + pos = SCOUTFS_I(dir)->next_readdir_pos++; + + ret = add_entry_items(sb, scoutfs_ino(dir), hash, pos, + dentry->d_name.name, dentry->d_name.len, + scoutfs_ino(inode), inode->i_mode, dir_lock, + inode_lock); + if (ret) + goto out; + + update_dentry_info(sb, dentry, hash, pos, dir_lock); + + i_size_write(dir, i_size_read(dir) + dentry->d_name.len); + dir->i_mtime = dir->i_ctime = CURRENT_TIME; + + inode->i_ctime = dir->i_mtime; + i_size_write(inode, name_len); + + scoutfs_update_inode_item(inode, inode_lock, &ind_locks); + scoutfs_update_inode_item(dir, dir_lock, &ind_locks); + + insert_inode_hash(inode); + /* XXX need to set i_op/fop before here for sec callbacks */ + d_instantiate(dentry, inode); +out: + if (ret < 0) { + /* XXX remove inode items */ + if (!IS_ERR_OR_NULL(inode)) + iput(inode); + + symlink_item_ops(sb, SYM_DELETE, scoutfs_ino(inode), inode_lock, + NULL, name_len); + } + + scoutfs_release_trans(sb); + scoutfs_inode_index_unlock(sb, &ind_locks); + scoutfs_unlock(sb, dir_lock, SCOUTFS_LOCK_WRITE); + scoutfs_unlock(sb, inode_lock, SCOUTFS_LOCK_WRITE); + + return ret; +} + +int scoutfs_symlink_drop(struct super_block *sb, u64 ino, + struct scoutfs_lock *lock, u64 i_size) +{ + int ret; + + ret = symlink_item_ops(sb, SYM_DELETE, ino, lock, NULL, i_size); + if (ret == -ENOENT) + ret = 0; + + return ret; +} + +/* + * Find the next link backref key for the given ino starting from the + * given dir inode and final entry position. If we find a backref item + * we add an allocated copy of it to the head of the caller's list. + * + * Returns 0 if we added an entry, -ENOENT if we didn't, and -errno for + * search errors. + * + * Callers are comfortable with the race inherent to incrementally + * building up a path with individual locked backref item lookups. + */ +int scoutfs_dir_add_next_linkref(struct super_block *sb, u64 ino, + u64 dir_ino, u64 dir_pos, + struct list_head *list) +{ + struct scoutfs_link_backref_entry *ent; + struct scoutfs_key last_key; + struct scoutfs_key key; + struct scoutfs_lock *lock = NULL; + int len; + int ret; + + ent = kmalloc(offsetof(struct scoutfs_link_backref_entry, + dent.name[SCOUTFS_NAME_LEN]), GFP_KERNEL); + if (!ent) { + ret = -ENOMEM; + goto out; + } + + INIT_LIST_HEAD(&ent->head); + + init_dirent_key(&key, SCOUTFS_LINK_BACKREF_TYPE, ino, dir_ino, dir_pos); + init_dirent_key(&last_key, SCOUTFS_LINK_BACKREF_TYPE, ino, U64_MAX, + U64_MAX); + + ret = scoutfs_lock_ino(sb, SCOUTFS_LOCK_READ, 0, ino, &lock); + if (ret) + goto out; + + ret = scoutfs_item_next(sb, &key, &last_key, &ent->dent, + dirent_bytes(SCOUTFS_NAME_LEN), lock); + scoutfs_unlock(sb, lock, SCOUTFS_LOCK_READ); + lock = NULL; + if (ret < 0) + goto out; + + len = ret - sizeof(struct scoutfs_dirent); + if (len < 1 || len > SCOUTFS_NAME_LEN) { + scoutfs_corruption(sb, SC_DIRENT_BACKREF_NAME_LEN, + corrupt_dirent_backref_name_len, + "ino %llu dir_ino %llu pos %llu key "SK_FMT" len %d", + ino, dir_ino, dir_pos, SK_ARG(&key), len); + ret = -EIO; + goto out; + } + + list_add(&ent->head, list); + ent->dir_ino = le64_to_cpu(key.skd_major); + ent->dir_pos = le64_to_cpu(key.skd_minor); + ent->name_len = len; + ret = 0; +out: + trace_scoutfs_dir_add_next_linkref(sb, ino, dir_ino, dir_pos, ret, + ent ? ent->dir_ino : 0, + ent ? ent->dir_pos : 0, + ent ? ent->name_len : 0); + + if (ent && list_empty(&ent->head)) + kfree(ent); + return ret; +} + +static u64 first_backref_dir_ino(struct list_head *list) +{ + struct scoutfs_link_backref_entry *ent; + + ent = list_first_entry(list, struct scoutfs_link_backref_entry, head); + return ent->dir_ino; +} + +void scoutfs_dir_free_backref_path(struct super_block *sb, + struct list_head *list) +{ + struct scoutfs_link_backref_entry *ent; + struct scoutfs_link_backref_entry *pos; + + list_for_each_entry_safe(ent, pos, list, head) { + list_del_init(&ent->head); + kfree(ent); + } +} + +/* + * Give the caller the next path from the root to the inode by walking + * backref items from the dir and name position, putting the backref keys + * we find in the caller's list. + * + * Return 0 if we found a path, -ENOENT if we didn't, and -errno on error. + * + * If parents get unlinked while we're searching we can fail to make it + * up to the root. We restart the search in that case. Parent dirs + * couldn't have been unlinked while they still had entries and we won't + * see links to the inode that have been unlinked. + * + * XXX Each path component traversal is consistent but that doesn't mean + * that the total traversed path is consistent. If renames hit dirs + * that have been visited and then dirs to be visited we can return a + * path that was never present in the system: + * + * path to inode mv performed built up path + * ---- + * a/b/c/d/e/f + * d/e/f + * mv a/b/c/d/e a/b/c/ + * a/b/c/e/f + * mv a/b/c a/ + * a/c/e/f + * a/c/d/e/f + * + * XXX We'll protect against this by sampling the seq before the + * traversal and restarting if we saw backref items whose seq was + * greater than the start point. It's not precise in that it doesn't + * also capture the rename of a dir that we already traversed but it + * lets us complete the traversal in one pass that very rarely restarts. + * + * XXX and worry about traversing entirely dirty backref items with + * equal seqs that have seen crazy modification? seems like we have to + * sync if we see our dirty seq. + */ +int scoutfs_dir_get_backref_path(struct super_block *sb, u64 ino, u64 dir_ino, + u64 dir_pos, struct list_head *list) +{ + int retries = 10; + u64 par_ino; + int ret; + +retry: + if (retries-- == 0) { + scoutfs_inc_counter(sb, dir_backref_excessive_retries); + ret = -ELOOP; + goto out; + } + + /* get the next link name to the given inode */ + ret = scoutfs_dir_add_next_linkref(sb, ino, dir_ino, dir_pos, list); + if (ret < 0) + goto out; + + /* then get the names of all the parent dirs */ + par_ino = first_backref_dir_ino(list); + while (par_ino != SCOUTFS_ROOT_INO) { + + ret = scoutfs_dir_add_next_linkref(sb, par_ino, 0, 0, list); + if (ret < 0) { + if (ret == -ENOENT) { + /* restart if there was no parent component */ + scoutfs_dir_free_backref_path(sb, list); + goto retry; + } + goto out; + } + + par_ino = first_backref_dir_ino(list); + } +out: + if (ret < 0) + scoutfs_dir_free_backref_path(sb, list); + return ret; +} + +/* + * Given two parent dir inos, return the ancestor of p2 that is p1's + * child when p1 is also an ancestor of p2: p1/p/[...]/p2. This can + * return p2. + * + * We do this by walking link backref items. Each entry can be thought + * of as a dirent stored at the target. So the parent dir is stored in + * the target. + * + * The caller holds the global rename lock and link backref walk locks + * each inode as it looks up backrefs. + */ +static int item_d_ancestor(struct super_block *sb, u64 p1, u64 p2, u64 *p_ret) +{ + struct scoutfs_link_backref_entry *ent; + LIST_HEAD(list); + int ret; + u64 p; + + *p_ret = 0; + + ret = scoutfs_dir_get_backref_path(sb, p2, 0, 0, &list); + if (ret) + goto out; + + p = p2; + list_for_each_entry(ent, &list, head) { + if (ent->dir_ino == p1) { + *p_ret = p; + ret = 0; + break; + } + p = ent->dir_ino; + } + +out: + scoutfs_dir_free_backref_path(sb, &list); + return ret; +} + +/* + * The vfs checked the relationship between dirs, the source, and target + * before acquiring clusters locks. All that could have changed. If + * we're renaming between parent dirs then we try to verify the basics + * of those checks using our backref items. + * + * Compare this to lock_rename()'s use of d_ancestor() and what it's + * caller does with the returned ancestor. + * + * The caller only holds the global rename cluster lock. + * item_d_ancestor is going to walk backref paths and acquire and + * release locks for each target inode in the path. + */ +static int verify_ancestors(struct super_block *sb, u64 p1, u64 p2, + u64 old_ino, u64 new_ino) +{ + int ret; + u64 p; + + ret = item_d_ancestor(sb, p1, p2, &p); + if (ret == 0 && p == 0) + ret = item_d_ancestor(sb, p2, p1, &p); + if (ret == 0 && p && (p == old_ino || p == new_ino)) + ret = -EINVAL; + + return ret; +} + +/* + * Make sure that a dirent from the dir to the inode exists at the name. + * The caller has the name locked in the dir. + */ +static int verify_entry(struct super_block *sb, u64 dir_ino, const char *name, + unsigned name_len, u64 hash, u64 ino, + struct scoutfs_lock *lock) +{ + struct scoutfs_dirent dent; + int ret; + + ret = lookup_dirent(sb, dir_ino, name, name_len, hash, &dent, lock); + if (ret == 0 && le64_to_cpu(dent.ino) != ino) + ret = -ENOENT; + else if (ret == -ENOENT && ino == 0) + ret = 0; + + return ret; +} + +/* + * The vfs performs checks on cached inodes and dirents before calling + * here. It doesn't hold any locks so all of those checks can be based + * on cached state that has been invalidated by other operations in the + * cluster before we get here. + * + * We do the expedient thing today and verify the basic structural + * checks after we get cluster locks. We perform topology checks + * analagous to the d_ancestor() walks in lock_rename() after acquiring + * a clustered equivalent of the vfs rename lock. We then lock the dir + * and target inodes and verify that the entries assumed by the function + * arguments still exist. + * + * We don't duplicate all the permissions checking in the vfs + * (may_create(), etc, are all static.). This means racing renames can + * succeed after other nodes have gotten success out of changes to + * permissions that should have forbidden renames. + * + * All of this wouldn't be necessary if we could get prepare/complete + * callbacks around rename that'd let us lock the inodes, dirents, and + * topology while the vfs walks dentries and uses inodes. + * + * We acquire the inode locks in inode number order. Because of our + * inode group locking we can't define lock ordering correctness by + * properties that can be different in a given group. This prevents us + * from using parent/child locking orders as two groups can have both + * parent and child relationships to each other. + */ +static int scoutfs_rename(struct inode *old_dir, struct dentry *old_dentry, + struct inode *new_dir, struct dentry *new_dentry) +{ + struct super_block *sb = old_dir->i_sb; + struct inode *old_inode = old_dentry->d_inode; + struct inode *new_inode = new_dentry->d_inode; + struct scoutfs_lock *rename_lock = NULL; + struct scoutfs_lock *old_dir_lock = NULL; + struct scoutfs_lock *new_dir_lock = NULL; + struct scoutfs_lock *old_inode_lock = NULL; + struct scoutfs_lock *new_inode_lock = NULL; + struct timespec now; + bool ins_new = false; + bool del_new = false; + bool ins_old = false; + LIST_HEAD(ind_locks); + u64 ind_seq; + u64 old_hash; + u64 new_hash; + u64 new_pos; + int ret; + int err; + + trace_scoutfs_rename(sb, old_dir, old_dentry, new_dir, new_dentry); + + old_hash = dirent_name_hash(old_dentry->d_name.name, + old_dentry->d_name.len); + new_hash = dirent_name_hash(new_dentry->d_name.name, + new_dentry->d_name.len); + + if (new_dentry->d_name.len > SCOUTFS_NAME_LEN) + return -ENAMETOOLONG; + + /* if dirs are different make sure ancestor relationships are valid */ + if (old_dir != new_dir) { + ret = scoutfs_lock_rename(sb, SCOUTFS_LOCK_WRITE, 0, + &rename_lock); + if (ret) + return ret; + + ret = verify_ancestors(sb, scoutfs_ino(old_dir), + scoutfs_ino(new_dir), + scoutfs_ino(old_inode), + new_inode ? scoutfs_ino(new_inode) : 0); + if (ret) + goto out_unlock; + } + + /* lock all the inodes */ + ret = scoutfs_lock_inodes(sb, SCOUTFS_LOCK_WRITE, + SCOUTFS_LKF_REFRESH_INODE, + old_dir, &old_dir_lock, + new_dir, &new_dir_lock, + old_inode, &old_inode_lock, + new_inode, &new_inode_lock); + if (ret) + goto out_unlock; + + /* test dir i_size now that it's refreshed */ + if (new_inode && S_ISDIR(new_inode->i_mode) && i_size_read(new_inode)) { + ret = -ENOTEMPTY; + goto out_unlock; + } + + /* make sure that the entries assumed by the argument still exist */ + ret = verify_entry(sb, scoutfs_ino(old_dir), old_dentry->d_name.name, + old_dentry->d_name.len, old_hash, + scoutfs_ino(old_inode), old_dir_lock) ?: + verify_entry(sb, scoutfs_ino(new_dir), new_dentry->d_name.name, + new_dentry->d_name.len, new_hash, + new_inode ? scoutfs_ino(new_inode) : 0, + new_dir_lock); + if (ret) + goto out_unlock; + +retry: + ret = scoutfs_inode_index_start(sb, &ind_seq) ?: + scoutfs_inode_index_prepare(sb, &ind_locks, old_dir, false) ?: + scoutfs_inode_index_prepare(sb, &ind_locks, old_inode, false) ?: + (new_dir == old_dir ? 0 : + scoutfs_inode_index_prepare(sb, &ind_locks, new_dir, false)) ?: + (new_inode == NULL ? 0 : + scoutfs_inode_index_prepare(sb, &ind_locks, new_inode, false)) ?: + scoutfs_inode_index_try_lock_hold(sb, &ind_locks, ind_seq, + SIC_RENAME(old_dentry->d_name.len, + new_dentry->d_name.len)); + if (ret > 0) + goto retry; + if (ret) + goto out_unlock; + + /* get a pos for the new entry */ + new_pos = SCOUTFS_I(new_dir)->next_readdir_pos++; + + /* dirty the inodes so that updating doesn't fail */ + ret = scoutfs_dirty_inode_item(old_dir, old_dir_lock) ?: + scoutfs_dirty_inode_item(old_inode, old_inode_lock) ?: + (old_dir != new_dir ? + scoutfs_dirty_inode_item(new_dir, new_dir_lock) : 0) ?: + (new_inode ? + scoutfs_dirty_inode_item(new_inode, new_inode_lock) : 0); + if (ret) + goto out; + + /* remove the new entry if it exists */ + if (new_inode) { + ret = del_entry_items(sb, scoutfs_ino(new_dir), + dentry_info_hash(new_dentry), + dentry_info_pos(new_dentry), + scoutfs_ino(new_inode), + new_dir_lock, new_inode_lock); + if (ret) + goto out; + ins_new = true; + } + + /* create the new entry */ + ret = add_entry_items(sb, scoutfs_ino(new_dir), new_hash, new_pos, + new_dentry->d_name.name, new_dentry->d_name.len, + scoutfs_ino(old_inode), old_inode->i_mode, + new_dir_lock, old_inode_lock); + if (ret) + goto out; + del_new = true; + + /* remove the old entry */ + ret = del_entry_items(sb, scoutfs_ino(old_dir), + dentry_info_hash(old_dentry), + dentry_info_pos(old_dentry), + scoutfs_ino(old_inode), + old_dir_lock, old_inode_lock); + if (ret) + goto out; + ins_old = true; + + if (should_orphan(new_inode)) { + ret = scoutfs_orphan_inode(new_inode); + if (ret) + goto out; + } + + /* won't fail from here on out, update all the vfs structs */ + + /* the caller will use d_move to move the old_dentry into place */ + update_dentry_info(sb, old_dentry, new_hash, new_pos, new_dir_lock); + + i_size_write(old_dir, i_size_read(old_dir) - old_dentry->d_name.len); + if (!new_inode) + i_size_write(new_dir, i_size_read(new_dir) + + new_dentry->d_name.len); + + if (new_inode) { + drop_nlink(new_inode); + if (S_ISDIR(new_inode->i_mode)) { + drop_nlink(new_dir); + drop_nlink(new_inode); + } + + } + + if (S_ISDIR(old_inode->i_mode) && (old_dir != new_dir)) { + drop_nlink(old_dir); + inc_nlink(new_dir); + } + + now = CURRENT_TIME; + old_dir->i_ctime = now; + old_dir->i_mtime = now; + if (new_dir != old_dir) { + new_dir->i_ctime = now; + new_dir->i_mtime = now; + } + old_inode->i_ctime = now; + if (new_inode) + old_inode->i_ctime = now; + + scoutfs_update_inode_item(old_dir, old_dir_lock, &ind_locks); + scoutfs_update_inode_item(old_inode, old_inode_lock, &ind_locks); + if (new_dir != old_dir) + scoutfs_update_inode_item(new_dir, new_dir_lock, &ind_locks); + if (new_inode) + scoutfs_update_inode_item(new_inode, new_inode_lock, + &ind_locks); + ret = 0; +out: + if (ret) { + /* + * XXX We have to clean up partial item deletions today + * because we can't have two dirents existing in a + * directory that point to different inodes. If we + * could we'd create the new name then everything after + * that is deletion that will only fail cleanly or + * succeed. Maybe we could have an item replace call + * that gives us the dupe to re-insert on cleanup? Not + * sure. + * + * It's safe to use dentry_info here 'cause they haven't + * been updated if we saw an error. + */ + err = 0; + if (ins_old) + err = add_entry_items(sb, scoutfs_ino(old_dir), + dentry_info_hash(old_dentry), + dentry_info_pos(old_dentry), + old_dentry->d_name.name, + old_dentry->d_name.len, + scoutfs_ino(old_inode), + old_inode->i_mode, + old_dir_lock, + old_inode_lock); + + if (del_new && err == 0) + err = del_entry_items(sb, scoutfs_ino(new_dir), + new_hash, new_pos, + scoutfs_ino(old_inode), + new_dir_lock, old_inode_lock); + + if (ins_new && err == 0) + err = add_entry_items(sb, scoutfs_ino(new_dir), + dentry_info_hash(new_dentry), + dentry_info_pos(new_dentry), + new_dentry->d_name.name, + new_dentry->d_name.len, + scoutfs_ino(new_inode), + new_inode->i_mode, + new_dir_lock, + new_inode_lock); + /* XXX freak out: panic, go read only, etc */ + BUG_ON(err); + } + + scoutfs_release_trans(sb); + +out_unlock: + scoutfs_inode_index_unlock(sb, &ind_locks); + scoutfs_unlock(sb, old_inode_lock, SCOUTFS_LOCK_WRITE); + scoutfs_unlock(sb, new_inode_lock, SCOUTFS_LOCK_WRITE); + scoutfs_unlock(sb, old_dir_lock, SCOUTFS_LOCK_WRITE); + scoutfs_unlock(sb, new_dir_lock, SCOUTFS_LOCK_WRITE); + scoutfs_unlock(sb, rename_lock, SCOUTFS_LOCK_WRITE); + + return ret; +} + +#ifdef KC_FMODE_KABI_ITERATE +/* we only need this to set the iterate flag for kabi :/ */ +static int scoutfs_dir_open(struct inode *inode, struct file *file) +{ + file->f_mode |= FMODE_KABI_ITERATE; + return 0; +} +#endif + +const struct file_operations scoutfs_dir_fops = { + .KC_FOP_READDIR = scoutfs_readdir, +#ifdef KC_FMODE_KABI_ITERATE + .open = scoutfs_dir_open, +#endif + .unlocked_ioctl = scoutfs_ioctl, + .fsync = scoutfs_file_fsync, + .llseek = generic_file_llseek, +}; + +const struct inode_operations scoutfs_dir_iops = { + .lookup = scoutfs_lookup, + .mknod = scoutfs_mknod, + .create = scoutfs_create, + .mkdir = scoutfs_mkdir, + .link = scoutfs_link, + .unlink = scoutfs_unlink, + .rmdir = scoutfs_unlink, + .rename = scoutfs_rename, + .getattr = scoutfs_getattr, + .setattr = scoutfs_setattr, + .setxattr = scoutfs_setxattr, + .getxattr = scoutfs_getxattr, + .listxattr = scoutfs_listxattr, + .removexattr = scoutfs_removexattr, + .symlink = scoutfs_symlink, + .permission = scoutfs_permission, +}; + +void scoutfs_dir_exit(void) +{ + if (dentry_info_cache) { + kmem_cache_destroy(dentry_info_cache); + dentry_info_cache = NULL; + } +} + +int scoutfs_dir_init(void) +{ + dentry_info_cache = kmem_cache_create("scoutfs_dentry_info", + sizeof(struct dentry_info), 0, + SLAB_RECLAIM_ACCOUNT, NULL); + if (!dentry_info_cache) + return -ENOMEM; + + return 0; +} diff --git a/kmod/src/dir.h b/kmod/src/dir.h new file mode 100644 index 00000000..ee43930e --- /dev/null +++ b/kmod/src/dir.h @@ -0,0 +1,35 @@ +#ifndef _SCOUTFS_DIR_H_ +#define _SCOUTFS_DIR_H_ + +#include "format.h" +#include "lock.h" + +extern const struct file_operations scoutfs_dir_fops; +extern const struct inode_operations scoutfs_dir_iops; +extern const struct inode_operations scoutfs_symlink_iops; + +struct scoutfs_link_backref_entry { + struct list_head head; + u64 dir_ino; + u64 dir_pos; + u16 name_len; + struct scoutfs_dirent dent; + /* the full name is allocated and stored in dent.name[0] */ +}; + +int scoutfs_dir_get_backref_path(struct super_block *sb, u64 ino, u64 dir_ino, + u64 dir_pos, struct list_head *list); +void scoutfs_dir_free_backref_path(struct super_block *sb, + struct list_head *list); + +int scoutfs_dir_add_next_linkref(struct super_block *sb, u64 ino, + u64 dir_ino, u64 dir_pos, + struct list_head *list); + +int scoutfs_symlink_drop(struct super_block *sb, u64 ino, + struct scoutfs_lock *lock, u64 i_size); + +int scoutfs_dir_init(void); +void scoutfs_dir_exit(void); + +#endif diff --git a/kmod/src/endian_swap.h b/kmod/src/endian_swap.h new file mode 100644 index 00000000..e64d119e --- /dev/null +++ b/kmod/src/endian_swap.h @@ -0,0 +1,12 @@ +#ifndef _SCOUTFS_ENDIAN_SWAP_H_ +#define _SCOUTFS_ENDIAN_SWAP_H_ + +#define le64_to_be64(x) cpu_to_be64(le64_to_cpu(x)) +#define le32_to_be32(x) cpu_to_be32(le32_to_cpu(x)) +#define le16_to_be16(x) cpu_to_be16(le16_to_cpu(x)) + +#define be64_to_le64(x) cpu_to_le64(be64_to_cpu(x)) +#define be32_to_le32(x) cpu_to_le32(be32_to_cpu(x)) +#define be16_to_le16(x) cpu_to_le16(be16_to_cpu(x)) + +#endif diff --git a/kmod/src/export.c b/kmod/src/export.c new file mode 100644 index 00000000..5ee59b41 --- /dev/null +++ b/kmod/src/export.c @@ -0,0 +1,167 @@ +/* + * Copyright (C) 2018 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ + +#include +#include +#include +#include +#include +#include + +#include "export.h" +#include "inode.h" +#include "dir.h" +#include "format.h" +#include "scoutfs_trace.h" + +/* describe the length of the fileid type in terms of number of u32's used. */ +static int scoutfs_fileid_len(int fh_type) +{ + switch (fh_type) { + case FILEID_SCOUTFS: + return 2; + case FILEID_SCOUTFS_WITH_PARENT: + return 4; + } + return FILEID_INVALID; +} + +static bool scoutfs_valid_fileid(int fh_type) +{ + return scoutfs_fileid_len(fh_type) != FILEID_INVALID; +} + +static int scoutfs_encode_fh(struct inode *inode, __u32 *fh, int *max_len, + struct inode *parent) +{ + struct scoutfs_fid *fid = (struct scoutfs_fid *)fh; + int fh_type = FILEID_SCOUTFS; + int len; + + if (parent) + fh_type = FILEID_SCOUTFS_WITH_PARENT; + + len = scoutfs_fileid_len(fh_type); + + if (*max_len < len) { + *max_len = len; + return FILEID_INVALID; + } + *max_len = len; + + fid->ino = cpu_to_le64(scoutfs_ino(inode)); + if (parent) + fid->parent_ino = cpu_to_le64(scoutfs_ino(parent)); + + trace_scoutfs_encode_fh(inode->i_sb, fh_type, fid); + + return fh_type; +} + +static struct dentry *scoutfs_fh_to_dentry(struct super_block *sb, + struct fid *fid, int fh_len, + int fh_type) +{ + struct scoutfs_fid *sfid = (struct scoutfs_fid *)fid; + struct inode *inode = NULL; + + if (fh_len < scoutfs_fileid_len(fh_type)) + return NULL; + + trace_scoutfs_fh_to_dentry(sb, fh_type, sfid); + + if (scoutfs_valid_fileid(fh_type)) + inode = scoutfs_iget(sb, le64_to_cpu(sfid->ino)); + + return d_obtain_alias(inode); +} + +static struct dentry *scoutfs_fh_to_parent(struct super_block *sb, + struct fid *fid, int fh_len, + int fh_type) +{ + struct scoutfs_fid *sfid = (struct scoutfs_fid *)fid; + struct inode *inode = NULL; + + if (fh_len < scoutfs_fileid_len(fh_type)) + return NULL; + + trace_scoutfs_fh_to_parent(sb, fh_type, sfid); + + if (scoutfs_valid_fileid(fh_type) && + fh_type == FILEID_SCOUTFS_WITH_PARENT) + inode = scoutfs_iget(sb, le64_to_cpu(sfid->parent_ino)); + + return d_obtain_alias(inode); +} + +static struct dentry *scoutfs_get_parent(struct dentry *child) +{ + struct inode *inode = child->d_inode; + struct super_block *sb = inode->i_sb; + struct scoutfs_link_backref_entry *ent; + LIST_HEAD(list); + int ret; + u64 ino; + + ret = scoutfs_dir_add_next_linkref(sb, scoutfs_ino(inode), 0, 0, &list); + if (ret) + return ERR_PTR(ret); + + ent = list_first_entry(&list, struct scoutfs_link_backref_entry, head); + ino = ent->dir_ino; + scoutfs_dir_free_backref_path(sb, &list); + trace_scoutfs_get_parent(sb, inode, ino); + + inode = scoutfs_iget(sb, ino); + + return d_obtain_alias(inode); +} + +static int scoutfs_get_name(struct dentry *parent, char *name, + struct dentry *child) +{ + u64 dir_ino = scoutfs_ino(parent->d_inode); + struct scoutfs_link_backref_entry *ent; + struct inode *inode = child->d_inode; + struct super_block *sb = inode->i_sb; + LIST_HEAD(list); + int ret; + + ret = scoutfs_dir_add_next_linkref(sb, scoutfs_ino(inode), dir_ino, + 0, &list); + if (ret) + return ret; + + ret = -ENOENT; + ent = list_first_entry(&list, struct scoutfs_link_backref_entry, head); + if (le64_to_cpu(ent->dent.ino) == scoutfs_ino(inode) && + ent->dir_ino == dir_ino && + ent->name_len <= NAME_MAX) { + memcpy(name, ent->dent.name, ent->name_len); + name[ent->name_len] = '\0'; + ret = 0; + trace_scoutfs_get_name(sb, parent->d_inode, inode, name); + } + scoutfs_dir_free_backref_path(sb, &list); + + return ret; +} + +const struct export_operations scoutfs_export_ops = { + .encode_fh = scoutfs_encode_fh, + .fh_to_dentry = scoutfs_fh_to_dentry, + .fh_to_parent = scoutfs_fh_to_parent, + .get_parent = scoutfs_get_parent, + .get_name = scoutfs_get_name, +}; diff --git a/kmod/src/export.h b/kmod/src/export.h new file mode 100644 index 00000000..7ed9771a --- /dev/null +++ b/kmod/src/export.h @@ -0,0 +1,8 @@ +#ifndef _SCOUTFS_EXPORT_H_ +#define _SCOUTFS_EXPORT_H_ + +#include + +extern const struct export_operations scoutfs_export_ops; + +#endif diff --git a/kmod/src/ext.c b/kmod/src/ext.c new file mode 100644 index 00000000..f87c064d --- /dev/null +++ b/kmod/src/ext.c @@ -0,0 +1,394 @@ +/* + * Copyright (C) 2020 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include + +#include "ext.h" +#include "counters.h" +#include "scoutfs_trace.h" + +/* + * Extents are used to track free block regions and to map logical file + * regions to device blocks. Extents can be split and merged as + * they're modified. These helpers implement all the fiddly extent + * manipulations. Callers provide callbacks which implement the actual + * storage of extents in either the item cache or btree items. + */ + +static void ext_zero(struct scoutfs_extent *ext) +{ + memset(ext, 0, sizeof(struct scoutfs_extent)); +} + +static bool ext_overlap(struct scoutfs_extent *ext, u64 start, u64 len) +{ + u64 e_end = ext->start + ext->len - 1; + u64 end = start + len - 1; + + return !(e_end < start || ext->start > end); +} + +static bool ext_inside(u64 start, u64 len, struct scoutfs_extent *out) +{ + u64 in_end = start + len - 1; + u64 out_end = out->start + out->len - 1; + + return out->start <= start && out_end >= in_end; +} + +/* we only translate mappings when they exist */ +static inline u64 ext_map_add(u64 map, u64 diff) +{ + return map ? map + diff : 0; +} + +/* + * Extents can merge if they're logically contiguous, both don't have + * mappings or have mappings which are also contiguous, and have + * matching flags. + */ +bool scoutfs_ext_can_merge(struct scoutfs_extent *left, + struct scoutfs_extent *right) +{ + return (left->start + left->len == right->start) && + ((!left->map && !right->map) || + (left->map + left->len == right->map)) && + (left->flags == right->flags); +} + +/* + * Split an existing extent in to left and right extents by removing + * an interior range. The split extents are all zeros if the range + * extends to their end of the extent. + */ +static void ext_split(struct scoutfs_extent *ext, u64 start, u64 len, + struct scoutfs_extent *left, + struct scoutfs_extent *right) +{ + if (ext->start < start) { + left->start = ext->start; + left->len = start - ext->start; + left->map = ext->map; + left->flags = ext->flags; + } else { + ext_zero(left); + } + + if (ext->start + ext->len > start + len) { + right->start = start + len; + right->len = ext->start + ext->len - right->start; + right->map = ext_map_add(ext->map, right->start - ext->start); + right->flags = ext->flags; + } else { + ext_zero(right); + } +} + +#define op_call(sb, ops, arg, which, args...) \ +({ \ + int _ret; \ + _ret = ops->which(sb, arg, ##args); \ + scoutfs_inc_counter(sb, ext_op_##which); \ + trace_scoutfs_ext_op_##which(sb, ##args, _ret); \ + _ret; \ +}) + +struct extent_changes { + struct scoutfs_extent exts[4]; + bool ins[4]; + u8 nr; +}; + +static void add_change(struct extent_changes *chg, + struct scoutfs_extent *ext, bool ins) +{ + BUILD_BUG_ON(ARRAY_SIZE(chg->ins) != ARRAY_SIZE(chg->exts)); + + if (ext->len) { + BUG_ON(chg->nr == ARRAY_SIZE(chg->exts)); + chg->exts[chg->nr] = *ext; + chg->ins[chg->nr] = !!ins; + chg->nr++; + } +} + +static int apply_changes(struct super_block *sb, struct scoutfs_ext_ops *ops, + void *arg, struct extent_changes *chg) +{ + int ret = 0; + int err; + int i; + + for (i = 0; i < chg->nr; i++) { + if (chg->ins[i]) + ret = op_call(sb, ops, arg, insert, chg->exts[i].start, + chg->exts[i].len, chg->exts[i].map, + chg->exts[i].flags); + else + ret = op_call(sb, ops, arg, remove, chg->exts[i].start, + chg->exts[i].len, chg->exts[i].map, + chg->exts[i].flags); + if (ret < 0) + break; + } + + while (ret < 0 && --i >= 0) { + if (chg->ins[i]) + err = op_call(sb, ops, arg, remove, chg->exts[i].start, + chg->exts[i].len, chg->exts[i].map, + chg->exts[i].flags); + else + err = op_call(sb, ops, arg, insert, chg->exts[i].start, + chg->exts[i].len, chg->exts[i].map, + chg->exts[i].flags); + BUG_ON(err); /* inconsistent */ + } + + return ret; +} + +int scoutfs_ext_next(struct super_block *sb, struct scoutfs_ext_ops *ops, + void *arg, u64 start, u64 len, struct scoutfs_extent *ext) +{ + int ret; + + ret = op_call(sb, ops, arg, next, start, len, ext); + trace_scoutfs_ext_next(sb, start, len, ext, ret); + return ret; +} + +/* + * Insert the given extent. EINVAL is returned if there's already an existing + * overlapping extent. This can merge with its neighbours. + */ +int scoutfs_ext_insert(struct super_block *sb, struct scoutfs_ext_ops *ops, + void *arg, u64 start, u64 len, u64 map, u8 flags) +{ + struct extent_changes chg = { .nr = 0 }; + struct scoutfs_extent found; + struct scoutfs_extent ins; + int ret; + + ins.start = start; + ins.len = len; + ins.map = map; + ins.flags = flags; + + /* find right neighbour and check for overlap */ + ret = op_call(sb, ops, arg, next, start, 1, &found); + if (ret < 0 && ret != -ENOENT) + goto out; + + /* inserting extent must not overlap */ + if (found.len && ext_overlap(&ins, found.start, found.len)) { + ret = -EINVAL; + goto out; + } + + /* merge with right if we can */ + if (found.len && scoutfs_ext_can_merge(&ins, &found)) { + ins.len += found.len; + add_change(&chg, &found, false); + } + + /* see if we can merge with a left neighbour */ + if (start > 0) { + ret = op_call(sb, ops, arg, next, start - 1, 1, &found); + if (ret < 0 && ret != -ENOENT) + goto out; + + if (ret == 0 && scoutfs_ext_can_merge(&found, &ins)) { + ins.start = found.start; + ins.map = found.map; + ins.len += found.len; + add_change(&chg, &found, false); + } + } + + add_change(&chg, &ins, true); + ret = apply_changes(sb, ops, arg, &chg); +out: + trace_scoutfs_ext_insert(sb, start, len, map, flags, ret); + return ret; +} + +/* + * Remove the given extent. The extent to remove must be found entirely + * in an existing extent. If the existing extent is larger then we leave + * behind the remaining extent. The existing extent can be split. + */ +int scoutfs_ext_remove(struct super_block *sb, struct scoutfs_ext_ops *ops, + void *arg, u64 start, u64 len) +{ + struct extent_changes chg = { .nr = 0 }; + struct scoutfs_extent found; + struct scoutfs_extent left; + struct scoutfs_extent right; + int ret; + + ret = op_call(sb, ops, arg, next, start, 1, &found); + if (ret < 0) + goto out; + + /* removed extent must be entirely within found */ + if (!ext_inside(start, len, &found)) { + ret = -EINVAL; + goto out; + } + + ext_split(&found, start, len, &left, &right); + + add_change(&chg, &found, false); + add_change(&chg, &left, true); + add_change(&chg, &right, true); + + ret = apply_changes(sb, ops, arg, &chg); +out: + trace_scoutfs_ext_remove(sb, start, len, 0, 0, ret); + return ret; +} + +/* + * Find and remove the next extent, removing only a portion if the + * extent is larger than the count. Returns ENOENT if it didn't + * find any extents. + * + * This does not search for merge candidates so it's safe to call with + * extents indexed by length. + */ +int scoutfs_ext_alloc(struct super_block *sb, struct scoutfs_ext_ops *ops, + void *arg, u64 start, u64 len, u64 count, + struct scoutfs_extent *ext) +{ + struct extent_changes chg = { .nr = 0 }; + struct scoutfs_extent found; + struct scoutfs_extent ins; + int ret; + + ret = op_call(sb, ops, arg, next, start, len, &found); + if (ret < 0) + goto out; + + add_change(&chg, &found, false); + + if (found.len > count) { + ins.start = found.start + count; + ins.len = found.len - count; + ins.map = ext_map_add(found.map, count); + ins.flags = found.flags; + + add_change(&chg, &ins, true); + } + + ret = apply_changes(sb, ops, arg, &chg); +out: + if (ret == 0) { + ext->start = found.start; + ext->len = min(found.len, count); + ext->map = found.map; + ext->flags = found.flags; + } else { + ext_zero(ext); + } + + trace_scoutfs_ext_alloc(sb, start, len, count, ext, ret); + return ret; +} + +/* + * Set the map and flags for an extent region, with the magical property + * that extents with map and flags set to 0 are removed. + * + * If we're modifying an existing extent then the modification must be + * fully inside the existing extent. The modification can leave edges + * of the extent which need to be inserted. If the modification extends + * to the end of the existing extent then we need to check for adjacent + * neighbouring extents which might now be able to be merged. + * + * Inserting a new extent is like the case of modifying the entire + * existing extent. We need to check neighbours of the inserted extent + * to see if they can be merged. + */ +int scoutfs_ext_set(struct super_block *sb, struct scoutfs_ext_ops *ops, + void *arg, u64 start, u64 len, u64 map, u8 flags) +{ + struct extent_changes chg = { .nr = 0 }; + struct scoutfs_extent found; + struct scoutfs_extent left; + struct scoutfs_extent right; + struct scoutfs_extent set; + int ret; + + set.start = start; + set.len = len; + set.map = map; + set.flags = flags; + + /* find extent to remove */ + ret = op_call(sb, ops, arg, next, start, 1, &found); + if (ret < 0 && ret != -ENOENT) + goto out; + + if (ret == 0 && ext_overlap(&found, start, len)) { + /* set extent must be entirely within found */ + if (!ext_inside(start, len, &found)) { + ret = -EINVAL; + goto out; + } + + add_change(&chg, &found, false); + ext_split(&found, start, len, &left, &right); + } else { + ext_zero(&found); + ext_zero(&left); + ext_zero(&right); + } + + if (left.len) { + /* inserting split left, won't merge */ + add_change(&chg, &left, true); + } else if (start > 0) { + ret = op_call(sb, ops, arg, next, start - 1, 1, &left); + if (ret < 0 && ret != -ENOENT) + goto out; + else if (ret == 0 && scoutfs_ext_can_merge(&left, &set)) { + /* remove found left, merging */ + set.start = left.start; + set.map = left.map; + set.len += left.len; + add_change(&chg, &left, false); + } + } + + if (right.len) { + /* inserting split right, won't merge */ + add_change(&chg, &right, true); + } else { + ret = op_call(sb, ops, arg, next, start + len, 1, &right); + if (ret < 0 && ret != -ENOENT) + goto out; + else if (ret == 0 && scoutfs_ext_can_merge(&set, &right)) { + /* remove found right, merging */ + set.len += right.len; + add_change(&chg, &right, false); + } + } + + if (set.flags || set.map) + add_change(&chg, &set, true); + + ret = apply_changes(sb, ops, arg, &chg); +out: + trace_scoutfs_ext_set(sb, start, len, map, flags, ret); + return ret; +} diff --git a/kmod/src/ext.h b/kmod/src/ext.h new file mode 100644 index 00000000..31dbd57a --- /dev/null +++ b/kmod/src/ext.h @@ -0,0 +1,35 @@ +#ifndef _SCOUTFS_EXT_H_ +#define _SCOUTFS_EXT_H_ + +struct scoutfs_extent { + u64 start; + u64 len; + u64 map; + u8 flags; +}; + +struct scoutfs_ext_ops { + int (*next)(struct super_block *sb, void *arg, + u64 start, u64 len, struct scoutfs_extent *ext); + int (*insert)(struct super_block *sb, void *arg, + u64 start, u64 len, u64 map, u8 flags); + int (*remove)(struct super_block *sb, void *arg, u64 start, u64 len, + u64 map, u8 flags); +}; + +bool scoutfs_ext_can_merge(struct scoutfs_extent *left, + struct scoutfs_extent *right); + +int scoutfs_ext_next(struct super_block *sb, struct scoutfs_ext_ops *ops, + void *arg, u64 start, u64 len, struct scoutfs_extent *ext); +int scoutfs_ext_insert(struct super_block *sb, struct scoutfs_ext_ops *ops, + void *arg, u64 start, u64 len, u64 map, u8 flags); +int scoutfs_ext_remove(struct super_block *sb, struct scoutfs_ext_ops *ops, + void *arg, u64 start, u64 len); +int scoutfs_ext_alloc(struct super_block *sb, struct scoutfs_ext_ops *ops, + void *arg, u64 start, u64 len, u64 limit, + struct scoutfs_extent *ext); +int scoutfs_ext_set(struct super_block *sb, struct scoutfs_ext_ops *ops, + void *arg, u64 start, u64 len, u64 map, u8 flags); + +#endif diff --git a/kmod/src/file.c b/kmod/src/file.c new file mode 100644 index 00000000..f35ef039 --- /dev/null +++ b/kmod/src/file.c @@ -0,0 +1,199 @@ +/* + * Copyright (C) 2017 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "format.h" +#include "super.h" +#include "data.h" +#include "scoutfs_trace.h" +#include "lock.h" +#include "file.h" +#include "inode.h" +#include "per_task.h" + +/* TODO: Direct I/O, AIO */ +ssize_t scoutfs_file_aio_read(struct kiocb *iocb, const struct iovec *iov, + unsigned long nr_segs, loff_t pos) +{ + struct file *file = iocb->ki_filp; + struct inode *inode = file_inode(file); + struct scoutfs_inode_info *si = SCOUTFS_I(inode); + struct super_block *sb = inode->i_sb; + struct scoutfs_lock *inode_lock = NULL; + SCOUTFS_DECLARE_PER_TASK_ENTRY(pt_ent); + DECLARE_DATA_WAIT(dw); + int ret; + +retry: + ret = scoutfs_lock_inode(sb, SCOUTFS_LOCK_READ, + SCOUTFS_LKF_REFRESH_INODE, inode, &inode_lock); + if (ret) + goto out; + + if (scoutfs_per_task_add_excl(&si->pt_data_lock, &pt_ent, inode_lock)) { + /* protect checked extents from stage/release */ + mutex_lock(&inode->i_mutex); + atomic_inc(&inode->i_dio_count); + mutex_unlock(&inode->i_mutex); + + ret = scoutfs_data_wait_check_iov(inode, iov, nr_segs, pos, + SEF_OFFLINE, + SCOUTFS_IOC_DWO_READ, + &dw, inode_lock); + if (ret != 0) + goto out; + } + + ret = generic_file_aio_read(iocb, iov, nr_segs, pos); + +out: + if (scoutfs_per_task_del(&si->pt_data_lock, &pt_ent)) + inode_dio_done(inode); + scoutfs_unlock(sb, inode_lock, SCOUTFS_LOCK_READ); + + if (scoutfs_data_wait_found(&dw)) { + ret = scoutfs_data_wait(inode, &dw); + if (ret == 0) + goto retry; + } + + return ret; +} + +ssize_t scoutfs_file_aio_write(struct kiocb *iocb, const struct iovec *iov, + unsigned long nr_segs, loff_t pos) +{ + struct file *file = iocb->ki_filp; + struct inode *inode = file_inode(file); + struct scoutfs_inode_info *si = SCOUTFS_I(inode); + struct super_block *sb = inode->i_sb; + struct scoutfs_lock *inode_lock = NULL; + SCOUTFS_DECLARE_PER_TASK_ENTRY(pt_ent); + DECLARE_DATA_WAIT(dw); + int ret; + + if (iocb->ki_left == 0) /* Does this even happen? */ + return 0; + +retry: + mutex_lock(&inode->i_mutex); + ret = scoutfs_lock_inode(sb, SCOUTFS_LOCK_WRITE, + SCOUTFS_LKF_REFRESH_INODE, inode, &inode_lock); + if (ret) + goto out; + + ret = scoutfs_complete_truncate(inode, inode_lock); + if (ret) + goto out; + + if (scoutfs_per_task_add_excl(&si->pt_data_lock, &pt_ent, inode_lock)) { + /* data_version is per inode, whole file must be online */ + ret = scoutfs_data_wait_check(inode, 0, i_size_read(inode), + SEF_OFFLINE, + SCOUTFS_IOC_DWO_WRITE, + &dw, inode_lock); + if (ret != 0) + goto out; + } + + /* XXX: remove SUID bit */ + + ret = __generic_file_aio_write(iocb, iov, nr_segs, &iocb->ki_pos); + +out: + scoutfs_per_task_del(&si->pt_data_lock, &pt_ent); + scoutfs_unlock(sb, inode_lock, SCOUTFS_LOCK_WRITE); + mutex_unlock(&inode->i_mutex); + + if (scoutfs_data_wait_found(&dw)) { + ret = scoutfs_data_wait(inode, &dw); + if (ret == 0) + goto retry; + } + + if (ret > 0 || ret == -EIOCBQUEUED) { + ssize_t err; + + err = generic_write_sync(file, pos, ret); + if (err < 0 && ret > 0) + ret = err; + } + + return ret; +} + +int scoutfs_permission(struct inode *inode, int mask) +{ + struct super_block *sb = inode->i_sb; + struct scoutfs_lock *inode_lock = NULL; + int ret; + + if (mask & MAY_NOT_BLOCK) + return -ECHILD; + + ret = scoutfs_lock_inode(sb, SCOUTFS_LOCK_READ, + SCOUTFS_LKF_REFRESH_INODE, inode, &inode_lock); + if (ret) + return ret; + + ret = generic_permission(inode, mask); + + scoutfs_unlock(sb, inode_lock, SCOUTFS_LOCK_READ); + + return ret; +} + +loff_t scoutfs_file_llseek(struct file *file, loff_t offset, int whence) +{ + struct inode *inode = file->f_mapping->host; + struct super_block *sb = inode->i_sb; + struct scoutfs_lock *lock = NULL; + int ret = 0; + + switch (whence) { + case SEEK_END: + case SEEK_DATA: + case SEEK_HOLE: + /* + * These require a lock and inode refresh as they + * reference i_size. + * + * XXX: SEEK_DATA/SEEK_HOLE can search our extent + * items instead of relying on generic_file_llseek() + * trickery. + */ + ret = scoutfs_lock_inode(sb, SCOUTFS_LOCK_READ, + SCOUTFS_LKF_REFRESH_INODE, inode, + &lock); + case SEEK_SET: + case SEEK_CUR: + /* No lock required, fall through to the generic helper */ + break; + default: + ret = -EINVAL; + } + + if (ret == 0) + offset = generic_file_llseek(file, offset, whence); + + scoutfs_unlock(sb, lock, SCOUTFS_LOCK_READ); + + return ret ? ret : offset; +} diff --git a/kmod/src/file.h b/kmod/src/file.h new file mode 100644 index 00000000..82d86618 --- /dev/null +++ b/kmod/src/file.h @@ -0,0 +1,11 @@ +#ifndef _SCOUTFS_FILE_H_ +#define _SCOUTFS_FILE_H_ + +ssize_t scoutfs_file_aio_read(struct kiocb *iocb, const struct iovec *iov, + unsigned long nr_segs, loff_t pos); +ssize_t scoutfs_file_aio_write(struct kiocb *iocb, const struct iovec *iov, + unsigned long nr_segs, loff_t pos); +int scoutfs_permission(struct inode *inode, int mask); +loff_t scoutfs_file_llseek(struct file *file, loff_t offset, int whence); + +#endif /* _SCOUTFS_FILE_H_ */ diff --git a/kmod/src/forest.c b/kmod/src/forest.c new file mode 100644 index 00000000..f5f259c0 --- /dev/null +++ b/kmod/src/forest.c @@ -0,0 +1,668 @@ +/* + * Copyright (C) 2019 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include +#include + +#include "super.h" +#include "format.h" +#include "lock.h" +#include "btree.h" +#include "client.h" +#include "alloc.h" +#include "block.h" +#include "forest.h" +#include "hash.h" +#include "srch.h" +#include "counters.h" +#include "scoutfs_trace.h" + +/* + * scoutfs items are stored in a forest of btrees. Each mount writes + * items into its own relatively small log btree. Each mount can also + * have a few finalized log btrees sitting around that it is no longer + * writing to. Finally a much larger core fs btree is the final home + * for metadata. + * + * The log btrees are modified by multiple transactions over time so + * there is no consistent ordering relationship between the items in + * different btrees. Each item in a log btree stores a version number + * for the item. Readers check log btrees for the most recent version + * that it should use. + * + * The item cache reads items in bulk from stable btrees, and writes a + * transaction's worth of dirty items into the item log btree. + * + * Log btrees are typically very sparse. It would be wasteful for + * readers to read every log btree looking for an item. Each log btree + * contains a bloom filter keyed on the starting key of locks. This + * lets lock holders quickly eliminate log trees that cannot contain + * keys protected by their lock. + */ + +struct forest_info { + struct mutex mutex; + struct scoutfs_alloc *alloc; + struct scoutfs_block_writer *wri; + struct scoutfs_log_trees our_log; + + struct mutex srch_mutex; + struct scoutfs_srch_file srch_file; + struct scoutfs_block *srch_bl; +}; + +#define DECLARE_FOREST_INFO(sb, name) \ + struct forest_info *name = SCOUTFS_SB(sb)->forest_info + +struct forest_refs { + struct scoutfs_btree_ref fs_ref; + struct scoutfs_btree_ref logs_ref; +}; + +/* initialize some refs that initially aren't equal */ +#define DECLARE_STALE_TRACKING_SUPER_REFS(a, b) \ + struct forest_refs a = {{cpu_to_le64(0),}}; \ + struct forest_refs b = {{cpu_to_le64(1),}} + +struct forest_bloom_nrs { + unsigned int nrs[SCOUTFS_FOREST_BLOOM_NRS]; +}; + +static void calc_bloom_nrs(struct forest_bloom_nrs *bloom, + struct scoutfs_key *key) +{ + u64 hash; + int i; + + BUILD_BUG_ON((SCOUTFS_FOREST_BLOOM_FUNC_BITS * + SCOUTFS_FOREST_BLOOM_NRS) > 64); + + hash = scoutfs_hash64(key, sizeof(struct scoutfs_key)); + + for (i = 0; i < ARRAY_SIZE(bloom->nrs); i++) { + bloom->nrs[i] = (u32)hash % SCOUTFS_FOREST_BLOOM_BITS; + hash >>= SCOUTFS_FOREST_BLOOM_FUNC_BITS; + } +} + +static struct scoutfs_block *read_bloom_ref(struct super_block *sb, + struct scoutfs_btree_ref *ref) +{ + struct scoutfs_block *bl; + + bl = scoutfs_block_read(sb, le64_to_cpu(ref->blkno)); + if (IS_ERR(bl)) + return bl; + + if (!scoutfs_block_consistent_ref(sb, bl, ref->seq, ref->blkno, + SCOUTFS_BLOCK_MAGIC_BLOOM)) { + scoutfs_block_invalidate(sb, bl); + scoutfs_block_put(sb, bl); + return ERR_PTR(-ESTALE); + } + + return bl; +} + +/* + * This is an unlocked iteration across all the btrees to find a hint at + * the next key that the caller could read. It's used to find out what + * next key range to lock, presuming you're allowed to only see items + * that have been synced. We ask the server for the current roots to + * check. + * + * We don't bother skipping deletion items here. The caller will safely + * skip over them when really reading from their locked region and will + * call again after them to find the next hint. + * + * We're reading from stable persistent trees so we don't need to lock + * against writers, their writes are cow into free blocks. + */ +int scoutfs_forest_next_hint(struct super_block *sb, struct scoutfs_key *key, + struct scoutfs_key *next) +{ + DECLARE_STALE_TRACKING_SUPER_REFS(prev_refs, refs); + struct scoutfs_net_roots roots; + struct scoutfs_btree_root item_root; + struct scoutfs_log_trees *lt; + SCOUTFS_BTREE_ITEM_REF(iref); + struct scoutfs_key found; + struct scoutfs_key ltk; + bool checked_fs; + bool have_next; + int ret; + + scoutfs_inc_counter(sb, forest_roots_next_hint); + +retry: + ret = scoutfs_client_get_roots(sb, &roots); + if (ret) + goto out; + + trace_scoutfs_forest_using_roots(sb, &roots.fs_root, &roots.logs_root); + refs.fs_ref = roots.fs_root.ref; + refs.logs_ref = roots.logs_root.ref; + + scoutfs_key_init_log_trees(<k, 0, 0); + checked_fs = false; + have_next = false; + + for (;;) { + if (!checked_fs) { + checked_fs = true; + item_root = roots.fs_root; + } else { + ret = scoutfs_btree_next(sb, &roots.logs_root, <k, + &iref); + if (ret == -ENOENT) { + if (have_next) + ret = 0; + break; + } + if (ret == -ESTALE) + break; + if (ret < 0) + goto out; + + if (iref.val_len == sizeof(*lt)) { + ltk = *iref.key; + scoutfs_key_inc(<k); + lt = iref.val; + item_root = lt->item_root; + } else { + ret = -EIO; + } + scoutfs_btree_put_iref(&iref); + if (ret < 0) + goto out; + + if (item_root.ref.blkno == 0) + continue; + } + + ret = scoutfs_btree_next(sb, &item_root, key, &iref); + if (ret == -ENOENT) + continue; + if (ret == -ESTALE) + break; + if (ret < 0) + goto out; + + found = *iref.key; + scoutfs_btree_put_iref(&iref); + + if (!have_next || scoutfs_key_compare(&found, next) < 0) { + have_next = true; + *next = found; + } + } + + if (ret == -ESTALE) { + if (memcmp(&prev_refs, &refs, sizeof(refs)) == 0) + return -EIO; + prev_refs = refs; + goto retry; + } +out: + + return ret; +} + +struct forest_read_items_data { + bool is_fs; + scoutfs_forest_item_cb cb; + void *cb_arg; +}; + +static int forest_read_items(struct super_block *sb, struct scoutfs_key *key, + void *val, int val_len, void *arg) +{ + struct forest_read_items_data *rid = arg; + struct scoutfs_log_item_value _liv = {0,}; + struct scoutfs_log_item_value *liv = &_liv; + + if (!rid->is_fs) { + liv = val; + val += sizeof(struct scoutfs_log_item_value); + val_len -= sizeof(struct scoutfs_log_item_value); + } + + return rid->cb(sb, key, liv, val, val_len, rid->cb_arg); +} + +/* + * For each forest btree whose bloom block indicates that the lock might + * have items stored, call the caller's callback for every item in the + * leaf block in each tree which contains the key. + * + * The btree iter calls clamp the caller's range to the tightest range + * that covers all the blocks. Any keys outside of this range can't be + * trusted because we didn't visit all the trees to check their items. + * + * If we hit stale blocks and retry we can call the callback for + * duplicate items. This is harmless because the items are stable while + * the caller holds their cluster lock and the caller has to filter out + * item versions anyway. + */ +int scoutfs_forest_read_items(struct super_block *sb, + struct scoutfs_lock *lock, + struct scoutfs_key *key, + struct scoutfs_key *start, + struct scoutfs_key *end, + scoutfs_forest_item_cb cb, void *arg) +{ + DECLARE_STALE_TRACKING_SUPER_REFS(prev_refs, refs); + struct forest_read_items_data rid = { + .cb = cb, + .cb_arg = arg, + }; + struct scoutfs_log_trees lt; + struct scoutfs_net_roots roots; + struct scoutfs_bloom_block *bb; + struct forest_bloom_nrs bloom; + SCOUTFS_BTREE_ITEM_REF(iref); + struct scoutfs_block *bl; + struct scoutfs_key ltk; + int ret; + int i; + + scoutfs_inc_counter(sb, forest_read_items); + calc_bloom_nrs(&bloom, &lock->start); + + roots = lock->roots; +retry: + ret = scoutfs_client_get_roots(sb, &roots); + if (ret) + goto out; + + trace_scoutfs_forest_using_roots(sb, &roots.fs_root, &roots.logs_root); + refs.fs_ref = roots.fs_root.ref; + refs.logs_ref = roots.logs_root.ref; + + *start = lock->start; + *end = lock->end; + + /* start with fs root items */ + rid.is_fs = true; + ret = scoutfs_btree_read_items(sb, &roots.fs_root, key, start, end, + forest_read_items, &rid); + if (ret < 0) + goto out; + rid.is_fs = false; + + scoutfs_key_init_log_trees(<k, 0, 0); + for (;; scoutfs_key_inc(<k)) { + ret = scoutfs_btree_next(sb, &roots.logs_root, <k, &iref); + if (ret == 0) { + if (iref.val_len == sizeof(lt)) { + ltk = *iref.key; + memcpy(<, iref.val, sizeof(lt)); + } else { + ret = -EIO; + } + scoutfs_btree_put_iref(&iref); + } + if (ret < 0) { + if (ret == -ENOENT) + break; + goto out; /* including stale */ + } + + if (lt.bloom_ref.blkno == 0) + continue; + + bl = read_bloom_ref(sb, <.bloom_ref); + if (IS_ERR(bl)) { + ret = PTR_ERR(bl); + goto out; + } + bb = bl->data; + + for (i = 0; i < ARRAY_SIZE(bloom.nrs); i++) { + if (!test_bit_le(bloom.nrs[i], bb->bits)) + break; + } + + scoutfs_block_put(sb, bl); + + /* one of the bloom bits wasn't set */ + if (i != ARRAY_SIZE(bloom.nrs)) { + scoutfs_inc_counter(sb, forest_bloom_fail); + continue; + } + + scoutfs_inc_counter(sb, forest_bloom_pass); + + ret = scoutfs_btree_read_items(sb, <.item_root, key, start, + end, forest_read_items, &rid); + if (ret < 0) + goto out; + } + + ret = 0; +out: + if (ret == -ESTALE) { + if (memcmp(&prev_refs, &refs, sizeof(refs)) == 0) { + ret = -EIO; + goto out; + } + prev_refs = refs; + + ret = scoutfs_client_get_roots(sb, &roots); + if (ret) + goto out; + goto retry; + } + + return ret; +} + +/* + * Make sure that the bloom bits for the lock's start key are all set in + * the current log's bloom block. We record the nr of our log tree in + * the lock so that we only try to cow and set the bits once per tree + * across multiple commits as long as the lock isn't purged. + * + * This is using a coarse mutex to serialize cowing the block. It could + * be much finer grained, but it's infrequent. We'll keep an eye on if + * it gets expensive enough to warrant fixing. + */ +int scoutfs_forest_set_bloom_bits(struct super_block *sb, + struct scoutfs_lock *lock) +{ + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + DECLARE_FOREST_INFO(sb, finf); + struct scoutfs_block *new_bl = NULL; + struct scoutfs_block *bl = NULL; + struct scoutfs_bloom_block *bb; + struct scoutfs_btree_ref *ref; + struct forest_bloom_nrs bloom; + int nr_set = 0; + u64 blkno; + u64 nr; + int ret; + int err; + int i; + + nr = le64_to_cpu(finf->our_log.nr); + + /* our rid is constant */ + if (atomic64_read(&lock->forest_bloom_nr) == nr) { + ret = 0; + goto out; + } + + mutex_lock(&finf->mutex); + + scoutfs_inc_counter(sb, forest_set_bloom_bits); + calc_bloom_nrs(&bloom, &lock->start); + + ref = &finf->our_log.bloom_ref; + + if (ref->blkno) { + bl = read_bloom_ref(sb, ref); + if (IS_ERR(bl)) { + ret = PTR_ERR(bl); + goto unlock; + } + bb = bl->data; + } + + if (!ref->blkno || !scoutfs_block_writer_is_dirty(sb, bl)) { + + ret = scoutfs_alloc_meta(sb, finf->alloc, finf->wri, &blkno); + if (ret < 0) + goto unlock; + + new_bl = scoutfs_block_create(sb, blkno); + if (IS_ERR(new_bl)) { + err = scoutfs_free_meta(sb, finf->alloc, finf->wri, + blkno); + BUG_ON(err); /* could have dirtied */ + ret = PTR_ERR(new_bl); + goto unlock; + } + + if (bl) { + err = scoutfs_free_meta(sb, finf->alloc, finf->wri, + le64_to_cpu(ref->blkno)); + BUG_ON(err); /* could have dirtied */ + memcpy(new_bl->data, bl->data, SCOUTFS_BLOCK_LG_SIZE); + } else { + memset(new_bl->data, 0, SCOUTFS_BLOCK_LG_SIZE); + } + + scoutfs_block_writer_mark_dirty(sb, finf->wri, new_bl); + + scoutfs_block_put(sb, bl); + bl = new_bl; + bb = bl->data; + new_bl = NULL; + + bb->hdr.magic = cpu_to_le32(SCOUTFS_BLOCK_MAGIC_BLOOM); + bb->hdr.fsid = super->hdr.fsid; + bb->hdr.blkno = cpu_to_le64(blkno); + prandom_bytes(&bb->hdr.seq, sizeof(bb->hdr.seq)); + ref->blkno = bb->hdr.blkno; + ref->seq = bb->hdr.seq; + } + + for (i = 0; i < ARRAY_SIZE(bloom.nrs); i++) { + if (!test_and_set_bit_le(bloom.nrs[i], bb->bits)) { + le64_add_cpu(&bb->total_set, 1); + nr_set++; + } + } + + trace_scoutfs_forest_bloom_set(sb, &lock->start, + le64_to_cpu(finf->our_log.rid), + le64_to_cpu(finf->our_log.nr), + le64_to_cpu(finf->our_log.bloom_ref.blkno), + le64_to_cpu(finf->our_log.bloom_ref.seq), + nr_set); + + atomic64_set(&lock->forest_bloom_nr, nr); + ret = 0; +unlock: + mutex_unlock(&finf->mutex); +out: + scoutfs_block_put(sb, bl); + return ret; +} + +/* + * The caller is commiting items in the transaction and has found the + * greatest item version amongst them. We store it in the log_trees root + * to send to the server. + */ +void scoutfs_forest_set_max_vers(struct super_block *sb, u64 max_vers) +{ + DECLARE_FOREST_INFO(sb, finf); + + finf->our_log.max_item_vers = cpu_to_le64(max_vers); +} + +/* + * The server is calling during setup to find the greatest item version + * amongst all the log tree roots. They have the authoritative current + * super. + * + * Item versions are only used to compare items in log trees, not in the + * main fs tree. All we have to do is find the greatest version amongst + * the log_trees so that new locks will have a write_version greater + * than all the items in the log_trees. + */ +int scoutfs_forest_get_max_vers(struct super_block *sb, + struct scoutfs_super_block *super, + u64 *vers) +{ + struct scoutfs_log_trees *lt; + SCOUTFS_BTREE_ITEM_REF(iref); + struct scoutfs_key ltk; + int ret; + + scoutfs_key_init_log_trees(<k, 0, 0); + *vers = 0; + + for (;; scoutfs_key_inc(<k)) { + ret = scoutfs_btree_next(sb, &super->logs_root, <k, &iref); + if (ret == 0) { + if (iref.val_len == sizeof(struct scoutfs_log_trees)) { + ltk = *iref.key; + lt = iref.val; + *vers = max(*vers, + le64_to_cpu(lt->max_item_vers)); + } else { + ret = -EIO; + } + scoutfs_btree_put_iref(&iref); + } + if (ret < 0) { + if (ret == -ENOENT) + break; + goto out; + } + } + + ret = 0; +out: + return ret; +} + +int scoutfs_forest_insert_list(struct super_block *sb, + struct scoutfs_btree_item_list *lst) +{ + DECLARE_FOREST_INFO(sb, finf); + + return scoutfs_btree_insert_list(sb, finf->alloc, finf->wri, + &finf->our_log.item_root, lst); +} + +/* + * Add a srch entry to the current transaction's log file. It will be + * committed in a transaction along with the dirty btree blocks that + * hold dirty items. The srch entries aren't governed by lock + * consistency. + * + * We lock here because of the shared file and block reference. + * Typically these calls are a quick appending to the end of the block, + * but they will allocate or cow blocks every few thousand calls. + */ +int scoutfs_forest_srch_add(struct super_block *sb, u64 hash, u64 ino, u64 id) +{ + DECLARE_FOREST_INFO(sb, finf); + int ret; + + mutex_lock(&finf->srch_mutex); + ret = scoutfs_srch_add(sb, finf->alloc, finf->wri, &finf->srch_file, + &finf->srch_bl, hash, ino, id); + mutex_unlock(&finf->srch_mutex); + return ret; +} + +/* + * This is called from transactions as a new transaction opens and is + * serialized with all writers. + */ +void scoutfs_forest_init_btrees(struct super_block *sb, + struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_log_trees *lt) +{ + DECLARE_FOREST_INFO(sb, finf); + + mutex_lock(&finf->mutex); + + finf->alloc = alloc; + finf->wri = wri; + + /* the lt allocator fields have been used by the caller */ + memset(&finf->our_log, 0, sizeof(finf->our_log)); + finf->our_log.item_root = lt->item_root; + finf->our_log.bloom_ref = lt->bloom_ref; + finf->our_log.max_item_vers = lt->max_item_vers; + finf->our_log.rid = lt->rid; + finf->our_log.nr = lt->nr; + finf->srch_file = lt->srch_file; + + WARN_ON_ONCE(finf->srch_bl); /* commiting should have put the block */ + finf->srch_bl = NULL; + + trace_scoutfs_forest_init_our_log(sb, le64_to_cpu(lt->rid), + le64_to_cpu(lt->nr), + le64_to_cpu(lt->item_root.ref.blkno), + le64_to_cpu(lt->item_root.ref.seq)); + + mutex_unlock(&finf->mutex); +} + +/* + * This is called during transaction commit which excludes forest writer + * calls. The caller has already written all the dirty blocks that the + * forest roots reference. They're getting the roots to send to the server + * for the commit. + */ +void scoutfs_forest_get_btrees(struct super_block *sb, + struct scoutfs_log_trees *lt) +{ + DECLARE_FOREST_INFO(sb, finf); + + lt->item_root = finf->our_log.item_root; + lt->bloom_ref = finf->our_log.bloom_ref; + lt->srch_file = finf->srch_file; + lt->max_item_vers = finf->our_log.max_item_vers; + + scoutfs_block_put(sb, finf->srch_bl); + finf->srch_bl = NULL; + + trace_scoutfs_forest_prepare_commit(sb, <->item_root.ref, + <->bloom_ref); +} + +int scoutfs_forest_setup(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct forest_info *finf; + int ret; + + finf = kzalloc(sizeof(struct forest_info), GFP_KERNEL); + if (!finf) { + ret = -ENOMEM; + goto out; + } + + /* the finf fields will be setup as we open a transaction */ + mutex_init(&finf->mutex); + mutex_init(&finf->srch_mutex); + + sbi->forest_info = finf; + ret = 0; +out: + if (ret) + scoutfs_forest_destroy(sb); + + return 0; +} + +void scoutfs_forest_destroy(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct forest_info *finf = SCOUTFS_SB(sb)->forest_info; + + if (finf) { + scoutfs_block_put(sb, finf->srch_bl); + kfree(finf); + sbi->forest_info = NULL; + } +} diff --git a/kmod/src/forest.h b/kmod/src/forest.h new file mode 100644 index 00000000..b73ea7a4 --- /dev/null +++ b/kmod/src/forest.h @@ -0,0 +1,44 @@ +#ifndef _SCOUTFS_FOREST_H_ +#define _SCOUTFS_FOREST_H_ + +struct scoutfs_alloc; +struct scoutfs_block_writer; +struct scoutfs_block; + +#include "btree.h" + +/* caller gives an item to the callback */ +typedef int (*scoutfs_forest_item_cb)(struct super_block *sb, + struct scoutfs_key *key, + struct scoutfs_log_item_value *liv, + void *val, int val_len, void *arg); + +int scoutfs_forest_next_hint(struct super_block *sb, struct scoutfs_key *key, + struct scoutfs_key *next); +int scoutfs_forest_read_items(struct super_block *sb, + struct scoutfs_lock *lock, + struct scoutfs_key *key, + struct scoutfs_key *start, + struct scoutfs_key *end, + scoutfs_forest_item_cb cb, void *arg); +int scoutfs_forest_set_bloom_bits(struct super_block *sb, + struct scoutfs_lock *lock); +void scoutfs_forest_set_max_vers(struct super_block *sb, u64 max_vers); +int scoutfs_forest_get_max_vers(struct super_block *sb, + struct scoutfs_super_block *super, + u64 *vers); +int scoutfs_forest_insert_list(struct super_block *sb, + struct scoutfs_btree_item_list *lst); +int scoutfs_forest_srch_add(struct super_block *sb, u64 hash, u64 ino, u64 id); + +void scoutfs_forest_init_btrees(struct super_block *sb, + struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_log_trees *lt); +void scoutfs_forest_get_btrees(struct super_block *sb, + struct scoutfs_log_trees *lt); + +int scoutfs_forest_setup(struct super_block *sb); +void scoutfs_forest_destroy(struct super_block *sb); + +#endif diff --git a/kmod/src/format.h b/kmod/src/format.h new file mode 100644 index 00000000..033552bf --- /dev/null +++ b/kmod/src/format.h @@ -0,0 +1,946 @@ +#ifndef _SCOUTFS_FORMAT_H_ +#define _SCOUTFS_FORMAT_H_ + +/* statfs(2) f_type */ +#define SCOUTFS_SUPER_MAGIC 0x554f4353 /* "SCOU" */ + +/* block header magic values, chosen at random */ +#define SCOUTFS_BLOCK_MAGIC_SUPER 0x103c428b +#define SCOUTFS_BLOCK_MAGIC_BTREE 0xe597f96d +#define SCOUTFS_BLOCK_MAGIC_BLOOM 0x31995604 +#define SCOUTFS_BLOCK_MAGIC_SRCH_BLOCK 0x897e4a7d +#define SCOUTFS_BLOCK_MAGIC_SRCH_PARENT 0xb23a2a05 +#define SCOUTFS_BLOCK_MAGIC_ALLOC_LIST 0x8a93ac83 + +/* + * The super block, quorum block, and file data allocation granularity + * use the smaller 4KB block. + */ +#define SCOUTFS_BLOCK_SM_SHIFT 12 +#define SCOUTFS_BLOCK_SM_SIZE (1 << SCOUTFS_BLOCK_SM_SHIFT) +#define SCOUTFS_BLOCK_SM_MASK (SCOUTFS_BLOCK_SM_SIZE - 1) +#define SCOUTFS_BLOCK_SM_PER_PAGE (PAGE_SIZE / SCOUTFS_BLOCK_SM_SIZE) +#define SCOUTFS_BLOCK_SM_SECTOR_SHIFT (SCOUTFS_BLOCK_SM_SHIFT - 9) +#define SCOUTFS_BLOCK_SM_SECTORS (1 << SCOUTFS_BLOCK_SM_SECTOR_SHIFT) +#define SCOUTFS_BLOCK_SM_MAX (U64_MAX >> SCOUTFS_BLOCK_SM_SHIFT) +#define SCOUTFS_BLOCK_SM_PAGES_PER (SCOUTFS_BLOCK_SM_SIZE / PAGE_SIZE) +#define SCOUTFS_BLOCK_SM_PAGE_ORDER (SCOUTFS_BLOCK_SM_SHIFT - PAGE_SHIFT) + +/* + * The radix and btree structures, and the forest bloom block, use the + * larger 64KB metadata block size. + */ +#define SCOUTFS_BLOCK_LG_SHIFT 16 +#define SCOUTFS_BLOCK_LG_SIZE (1 << SCOUTFS_BLOCK_LG_SHIFT) +#define SCOUTFS_BLOCK_LG_MASK (SCOUTFS_BLOCK_LG_SIZE - 1) +#define SCOUTFS_BLOCK_LG_PER_PAGE (PAGE_SIZE / SCOUTFS_BLOCK_LG_SIZE) +#define SCOUTFS_BLOCK_LG_SECTOR_SHIFT (SCOUTFS_BLOCK_LG_SHIFT - 9) +#define SCOUTFS_BLOCK_LG_SECTORS (1 << SCOUTFS_BLOCK_LG_SECTOR_SHIFT) +#define SCOUTFS_BLOCK_LG_MAX (U64_MAX >> SCOUTFS_BLOCK_LG_SHIFT) +#define SCOUTFS_BLOCK_LG_PAGES_PER (SCOUTFS_BLOCK_LG_SIZE / PAGE_SIZE) +#define SCOUTFS_BLOCK_LG_PAGE_ORDER (SCOUTFS_BLOCK_LG_SHIFT - PAGE_SHIFT) + +#define SCOUTFS_BLOCK_SM_LG_SHIFT (SCOUTFS_BLOCK_LG_SHIFT - \ + SCOUTFS_BLOCK_SM_SHIFT) + + +/* + * The super block leaves some room before the first block for platform + * structures like boot loaders. + */ +#define SCOUTFS_SUPER_BLKNO ((64ULL * 1024) >> SCOUTFS_BLOCK_SM_SHIFT) + +/* + * A reasonably large region of aligned quorum blocks follow the super + * block. Each voting cycle reads the entire region so we don't want it + * to be too enormous. 256K seems like a reasonably chunky single IO. + * The number of blocks in the region also determines the number of + * mounts that have a reasonable probability of not overwriting each + * other's random block locations. + */ +#define SCOUTFS_QUORUM_BLKNO ((256ULL * 1024) >> SCOUTFS_BLOCK_SM_SHIFT) +#define SCOUTFS_QUORUM_BLOCKS ((256ULL * 1024) >> SCOUTFS_BLOCK_SM_SHIFT) + +/* + * Start data on the data device aligned as well. + */ +#define SCOUTFS_DATA_DEV_START_BLKNO ((256ULL * 1024) >> SCOUTFS_BLOCK_SM_SHIFT) + + +#define SCOUTFS_UNIQUE_NAME_MAX_BYTES 64 /* includes null */ + +/* + * Base types used by other structures. + */ +struct scoutfs_timespec { + __le64 sec; + __le32 nsec; + __u8 __pad[4]; +}; + +/* XXX ipv6 */ +struct scoutfs_inet_addr { + __le32 addr; + __le16 port; + __u8 __pad[2]; +}; + +/* + * This header is stored at the start of btree blocks and the super + * block for verification. The crc field is not included in the + * calculation of the crc. + */ +struct scoutfs_block_header { + __le32 crc; + __le32 magic; + __le64 fsid; + __le64 seq; + __le64 blkno; +}; + +/* + * scoutfs identifies all file system metadata items by a small key + * struct. + * + * Each item type maps their logical structures to the fixed fields in + * sort order. This lets us print keys without needing per-type + * formats. + * + * The keys are compared by considering the fields in struct order from + * most to least significant. They are considered a multi precision + * value when navigating the keys in ordered key space. We can + * increment them, subtract them from each other, etc. + */ +struct scoutfs_key { + __le64 _sk_first; + __le64 _sk_second; + __le64 _sk_third; + __u8 _sk_fourth; + __u8 sk_zone; + __u8 sk_type; + __u8 __pad[5]; +}; + +/* inode index */ +#define skii_major _sk_second +#define skii_ino _sk_third + +/* node orphan inode */ +#define sko_rid _sk_first +#define sko_ino _sk_second + +/* inode */ +#define ski_ino _sk_first + +/* xattr parts */ +#define skx_ino _sk_first +#define skx_name_hash _sk_second +#define skx_id _sk_third +#define skx_part _sk_fourth + +/* directory entries */ +#define skd_ino _sk_first +#define skd_major _sk_second +#define skd_minor _sk_third + +/* symlink target */ +#define sks_ino _sk_first +#define sks_nr _sk_second + +/* data extents */ +#define skdx_ino _sk_first +#define skdx_end _sk_second +#define skdx_len _sk_third + +/* log trees */ +#define sklt_rid _sk_first +#define sklt_nr _sk_second + +/* lock clients */ +#define sklc_rid _sk_first + +/* seqs */ +#define skts_trans_seq _sk_first +#define skts_rid _sk_second + +/* mounted clients */ +#define skmc_rid _sk_first + +/* free extents by blkno */ +#define skfb_end _sk_second +#define skfb_len _sk_third +/* free extents by len */ +#define skfl_neglen _sk_second +#define skfl_blkno _sk_third + +struct scoutfs_radix_block { + struct scoutfs_block_header hdr; + union { + struct scoutfs_radix_ref { + __le64 blkno; + __le64 seq; + __le64 sm_total; + __le64 lg_total; + } refs[0]; + __le64 bits[0]; + }; +}; + +struct scoutfs_avl_root { + __le16 node; +}; + +struct scoutfs_avl_node { + __le16 parent; + __le16 left; + __le16 right; + __u8 height; + __u8 __pad[1]; +}; + +/* when we split we want to have multiple items on each side */ +#define SCOUTFS_BTREE_MAX_VAL_LEN 896 + +/* + * A 4EB test image measured a worst case height of 17. This is plenty + * generous. + */ +#define SCOUTFS_BTREE_MAX_HEIGHT 20 + +struct scoutfs_btree_ref { + __le64 blkno; + __le64 seq; +}; + +/* + * A height of X means that the first block read will have level X-1 and + * the leaves will have level 0. + */ +struct scoutfs_btree_root { + struct scoutfs_btree_ref ref; + __u8 height; + __u8 __pad[7]; +}; + +struct scoutfs_btree_item { + struct scoutfs_avl_node node; + struct scoutfs_key key; + __le16 val_off; + __le16 val_len; + __u8 __pad[4]; +}; + +struct scoutfs_btree_block { + struct scoutfs_block_header hdr; + struct scoutfs_avl_root item_root; + __le16 nr_items; + __le16 total_item_bytes; + __le16 mid_free_len; + __u8 level; + __u8 __pad[7]; + struct scoutfs_btree_item items[0]; + /* leaf blocks have a fixed size item offset hash table at the end */ +}; + +#define SCOUTFS_BTREE_VALUE_ALIGN 8 + +/* + * Try to aim for a 75% load in a leaf full of items with no value. + * We'll almost never see this because most items have values and most + * blocks aren't full. + */ +#define SCOUTFS_BTREE_LEAF_ITEM_HASH_NR_UNALIGNED \ + ((SCOUTFS_BLOCK_LG_SIZE - sizeof(struct scoutfs_btree_block)) / \ + (sizeof(struct scoutfs_btree_item) + (sizeof(__le16))) * 100 / 75) +#define SCOUTFS_BTREE_LEAF_ITEM_HASH_NR \ + (round_up(SCOUTFS_BTREE_LEAF_ITEM_HASH_NR_UNALIGNED, \ + SCOUTFS_BTREE_VALUE_ALIGN)) +#define SCOUTFS_BTREE_LEAF_ITEM_HASH_BYTES \ + (SCOUTFS_BTREE_LEAF_ITEM_HASH_NR * sizeof(__le16)) + +struct scoutfs_alloc_list_ref { + __le64 blkno; + __le64 seq; +}; + +/* + * first_nr tracks the nr of the first block in the list and is used for + * allocation sizing. total_nr is the sum of the nr of all the blocks in + * the list and is used for calculating total free block counts. + */ +struct scoutfs_alloc_list_head { + struct scoutfs_alloc_list_ref ref; + __le64 total_nr; + __le32 first_nr; + __u8 __pad[4]; +}; + +/* + * While the main allocator uses extent items in btree blocks, metadata + * allocations for a single transaction are recorded in arrays in + * blocks. This limits the number of allocations and frees needed to + * cow and modify the structure. The blocks can be stored in a list + * which lets us create a persistent log of pending frees that are + * generated as we cow btree blocks to insert freed extents. + * + * The array floats in the block so that both adding and removing blknos + * only modifies an index. + */ +struct scoutfs_alloc_list_block { + struct scoutfs_block_header hdr; + struct scoutfs_alloc_list_ref next; + __le32 start; + __le32 nr; + __le64 blknos[0]; /* naturally aligned for sorting */ +}; + +#define SCOUTFS_ALLOC_LIST_MAX_BLOCKS \ + ((SCOUTFS_BLOCK_LG_SIZE - sizeof(struct scoutfs_alloc_list_block)) / \ + (member_sizeof(struct scoutfs_alloc_list_block, blknos[0]))) + +/* + * These can safely be initialized to all-zeros. + */ +struct scoutfs_alloc_root { + __le64 total_len; + struct scoutfs_btree_root root; +}; + +/* types of allocators, exposed to alloc_detail ioctl */ +#define SCOUTFS_ALLOC_OWNER_NONE 0 +#define SCOUTFS_ALLOC_OWNER_SERVER 1 +#define SCOUTFS_ALLOC_OWNER_MOUNT 2 +#define SCOUTFS_ALLOC_OWNER_SRCH 3 + +struct scoutfs_mounted_client_btree_val { + __u8 flags; +}; + +#define SCOUTFS_MOUNTED_CLIENT_VOTER (1 << 0) + +/* + * srch files are a contiguous run of blocks with compressed entries + * described by a dense parent radix. The files can be stored in + * log_tree items when the files contain unsorted entries written by + * mounts during their transactions. Sorted files of increasing size + * are kept in a btree off the super for searching and further + * compacting. + */ +struct scoutfs_srch_entry { + __le64 hash; + __le64 ino; + __le64 id; +}; + +#define SCOUTFS_SRCH_ENTRY_MAX_BYTES (2 + (sizeof(__u64) * 3)) + +struct scoutfs_srch_ref { + __le64 blkno; + __le64 seq; +}; + +struct scoutfs_srch_file { + struct scoutfs_srch_entry first; + struct scoutfs_srch_entry last; + struct scoutfs_srch_ref ref; + __le64 blocks; + __le64 entries; + __u8 height; + __u8 __pad[7]; +}; + +struct scoutfs_srch_parent { + struct scoutfs_block_header hdr; + struct scoutfs_srch_ref refs[0]; +}; + +#define SCOUTFS_SRCH_PARENT_REFS \ + ((SCOUTFS_BLOCK_LG_SIZE - \ + offsetof(struct scoutfs_srch_parent, refs)) / \ + sizeof(struct scoutfs_srch_ref)) + +struct scoutfs_srch_block { + struct scoutfs_block_header hdr; + struct scoutfs_srch_entry first; + struct scoutfs_srch_entry last; + struct scoutfs_srch_entry tail; + __le32 entry_nr; + __le32 entry_bytes; + __u8 entries[0]; +}; + +/* + * Decoding loads final small deltas with full __u64 loads. Rather than + * check the size before each load we stop coding entries past the point + * where a full size entry could overflow the block. A final entry can + * start at this byte count and consume the rest of the block, though + * its unlikely. + */ +#define SCOUTFS_SRCH_BLOCK_SAFE_BYTES \ + (SCOUTFS_BLOCK_LG_SIZE - sizeof(struct scoutfs_srch_block) - \ + SCOUTFS_SRCH_ENTRY_MAX_BYTES) + +#define SCOUTFS_SRCH_LOG_BLOCK_LIMIT (1024 * 1024 / SCOUTFS_BLOCK_LG_SIZE) +#define SCOUTFS_SRCH_COMPACT_ORDER 2 +#define SCOUTFS_SRCH_COMPACT_NR (1 << SCOUTFS_SRCH_COMPACT_ORDER) + +/* + * A persistent record of a srch file compaction operation in progress. + * + * When compacting log files blk and pos aren't used. When compacting + * sorted files blk is the logical block number and pos is the byte + * offset of the next entry. When deleting files pos is the height of + * the level that we're deleting, and blk is the logical block offset of + * the next parent ref array index to descend through. + */ +struct scoutfs_srch_compact { + struct scoutfs_alloc_list_head meta_avail; + struct scoutfs_alloc_list_head meta_freed; + __le64 id; + __u8 nr; + __u8 flags; + __u8 __pad[6]; + struct scoutfs_srch_file out; + struct scoutfs_srch_compact_input { + struct scoutfs_srch_file sfl; + __le64 blk; + __le64 pos; + } in[SCOUTFS_SRCH_COMPACT_NR]; +}; + +/* server -> client: combine input log file entries into output file */ +#define SCOUTFS_SRCH_COMPACT_FLAG_LOG (1 << 0) +/* server -> client: combine input sorted file entries into output file */ +#define SCOUTFS_SRCH_COMPACT_FLAG_SORTED (1 << 1) +/* server -> client: delete input files */ +#define SCOUTFS_SRCH_COMPACT_FLAG_DELETE (1 << 2) +/* client -> server: compaction phase (LOG,SORTED,DELETE) done */ +#define SCOUTFS_SRCH_COMPACT_FLAG_DONE (1 << 4) +/* client -> server: compaction failed */ +#define SCOUTFS_SRCH_COMPACT_FLAG_ERROR (1 << 5) + +/* + * XXX I imagine we should rename these now that they've evolved to track + * all the btrees that clients use during a transaction. It's not just + * about item logs, it's about clients making changes to trees. + */ +struct scoutfs_log_trees { + struct scoutfs_alloc_list_head meta_avail; + struct scoutfs_alloc_list_head meta_freed; + struct scoutfs_btree_root item_root; + struct scoutfs_btree_ref bloom_ref; + struct scoutfs_alloc_root data_avail; + struct scoutfs_alloc_root data_freed; + struct scoutfs_srch_file srch_file; + __le64 max_item_vers; + __le64 rid; + __le64 nr; +}; + +struct scoutfs_log_item_value { + __le64 vers; + __u8 flags; + __u8 __pad[7]; + __u8 data[0]; +}; + +/* + * FS items are limited by the max btree value length with the log item + * value header. + */ +#define SCOUTFS_MAX_VAL_SIZE \ + (SCOUTFS_BTREE_MAX_VAL_LEN - sizeof(struct scoutfs_log_item_value)) + +#define SCOUTFS_LOG_ITEM_FLAG_DELETION (1 << 0) + +struct scoutfs_bloom_block { + struct scoutfs_block_header hdr; + __le64 total_set; + __le64 bits[0]; +}; + +/* + * Item log trees are accompanied by a block of bits that make up a + * bloom filter which indicate if the item log trees may contain items + * covered by a lock. The log trees should be finalized and merged long + * before the bloom filters fill up and start returning excessive false + * positives. + */ +#define SCOUTFS_FOREST_BLOOM_NRS 3 +#define SCOUTFS_FOREST_BLOOM_BITS \ + (((SCOUTFS_BLOCK_LG_SIZE - sizeof(struct scoutfs_bloom_block)) / \ + member_sizeof(struct scoutfs_bloom_block, bits[0])) * \ + member_sizeof(struct scoutfs_bloom_block, bits[0]) * 8) +#define SCOUTFS_FOREST_BLOOM_FUNC_BITS (SCOUTFS_BLOCK_LG_SHIFT + 3) + +/* + * Keys are first sorted by major key zones. + */ +#define SCOUTFS_INODE_INDEX_ZONE 1 +#define SCOUTFS_RID_ZONE 2 +#define SCOUTFS_FS_ZONE 3 +#define SCOUTFS_LOCK_ZONE 4 +/* Items only stored in server btrees */ +#define SCOUTFS_LOG_TREES_ZONE 6 +#define SCOUTFS_LOCK_CLIENTS_ZONE 7 +#define SCOUTFS_TRANS_SEQ_ZONE 8 +#define SCOUTFS_MOUNTED_CLIENT_ZONE 9 +#define SCOUTFS_SRCH_ZONE 10 +#define SCOUTFS_FREE_EXTENT_ZONE 11 + +/* inode index zone */ +#define SCOUTFS_INODE_INDEX_META_SEQ_TYPE 1 +#define SCOUTFS_INODE_INDEX_DATA_SEQ_TYPE 2 +#define SCOUTFS_INODE_INDEX_NR 3 /* don't forget to update */ + +/* rid zone (also used in server alloc btree) */ +#define SCOUTFS_ORPHAN_TYPE 1 + +/* fs zone */ +#define SCOUTFS_INODE_TYPE 1 +#define SCOUTFS_XATTR_TYPE 2 +#define SCOUTFS_DIRENT_TYPE 3 +#define SCOUTFS_READDIR_TYPE 4 +#define SCOUTFS_LINK_BACKREF_TYPE 5 +#define SCOUTFS_SYMLINK_TYPE 6 +#define SCOUTFS_DATA_EXTENT_TYPE 7 + +/* lock zone, only ever found in lock ranges, never in persistent items */ +#define SCOUTFS_RENAME_TYPE 1 + +/* srch zone, only in server btrees */ +#define SCOUTFS_SRCH_LOG_TYPE 1 +#define SCOUTFS_SRCH_BLOCKS_TYPE 2 +#define SCOUTFS_SRCH_PENDING_TYPE 3 +#define SCOUTFS_SRCH_BUSY_TYPE 4 + +/* free extents in allocator btrees in client and server, by blkno or len */ +#define SCOUTFS_FREE_EXTENT_BLKNO_TYPE 1 +#define SCOUTFS_FREE_EXTENT_LEN_TYPE 2 + +/* file data extents have start and len in key */ +struct scoutfs_data_extent_val { + __le64 blkno; + __u8 flags; + __u8 __pad[7]; +}; + +#define SEF_OFFLINE (1 << 0) +#define SEF_UNWRITTEN (1 << 1) +#define SEF_UNKNOWN (U8_MAX << 2) + +/* + * The first xattr part item has a header that describes the xattr. The + * name and value are then packed into the following bytes in the first + * part item and overflow into the values of the rest of the part items. + */ +struct scoutfs_xattr { + __le16 val_len; + __u8 name_len; + __u8 __pad[5]; + __u8 name[0]; +}; + + +/* XXX does this exist upstream somewhere? */ +#define member_sizeof(TYPE, MEMBER) (sizeof(((TYPE *)0)->MEMBER)) + +#define SCOUTFS_UUID_BYTES 16 + +/* + * Mounts read all the quorum blocks and write to one random quorum + * block during a cycle. The min cycle time limits the per-mount iop + * load during elections. The random cycle delay makes it less likely + * that mounts will read and write at the same time and miss each + * other's writes. An election only completes if a quorum of mounts + * vote for a leader before any of their elections timeout. This is + * made less likely by the probability that mounts will overwrite each + * others random block locations. The max quorum count limits that + * probability. 9 mounts only have a 55% chance of writing to unique 4k + * blocks in a 256k region. The election timeout is set to include + * enough cycles to usually complete the election. Once a leader is + * elected it spends a number of cycles writing out blocks with itself + * logged as a leader. This reduces the possibility that servers + * will have their log entries overwritten and not be fenced. + */ +#define SCOUTFS_QUORUM_MAX_COUNT 9 +#define SCOUTFS_QUORUM_CYCLE_LO_MS 10 +#define SCOUTFS_QUORUM_CYCLE_HI_MS 20 +#define SCOUTFS_QUORUM_TERM_LO_MS 250 +#define SCOUTFS_QUORUM_TERM_HI_MS 500 +#define SCOUTFS_QUORUM_ELECTED_LOG_CYCLES 10 + +struct scoutfs_quorum_block { + __le64 fsid; + __le64 blkno; + __le64 term; + __le64 write_nr; + __le64 voter_rid; + __le64 vote_for_rid; + __le32 crc; + __u8 log_nr; + __u8 __pad[3]; + struct scoutfs_quorum_log { + __le64 term; + __le64 rid; + struct scoutfs_inet_addr addr; + } log[0]; +}; + +#define SCOUTFS_QUORUM_LOG_MAX \ + ((SCOUTFS_BLOCK_SM_SIZE - sizeof(struct scoutfs_quorum_block)) / \ + sizeof(struct scoutfs_quorum_log)) + +#define SCOUTFS_FLAG_IS_META_BDEV 0x01 + +struct scoutfs_super_block { + struct scoutfs_block_header hdr; + __le64 id; + __le64 format_hash; + __le64 flags; + __u8 uuid[SCOUTFS_UUID_BYTES]; + __le64 next_ino; + __le64 next_trans_seq; + __le64 total_meta_blocks; /* both static and dynamic */ + __le64 first_meta_blkno; /* first dynamically allocated */ + __le64 last_meta_blkno; + __le64 total_data_blocks; + __le64 first_data_blkno; + __le64 last_data_blkno; + __le64 quorum_fenced_term; + __le64 quorum_server_term; + __le64 unmount_barrier; + __u8 quorum_count; + __u8 __pad[7]; + struct scoutfs_inet_addr server_addr; + struct scoutfs_alloc_root meta_alloc[2]; + struct scoutfs_alloc_root data_alloc; + struct scoutfs_alloc_list_head server_meta_avail[2]; + struct scoutfs_alloc_list_head server_meta_freed[2]; + struct scoutfs_btree_root fs_root; + struct scoutfs_btree_root logs_root; + struct scoutfs_btree_root lock_clients; + struct scoutfs_btree_root trans_seqs; + struct scoutfs_btree_root mounted_clients; + struct scoutfs_btree_root srch_root; +}; + +#define SCOUTFS_ROOT_INO 1 + + +/* + * @meta_seq: advanced the first time an inode is updated in a given + * transaction. It can only advance again after the inode is written + * and a new transaction opens. + * + * @data_seq: advanced the first time a file's data (or size) is + * modified in a given transaction. It can only advance again after the + * file is written and a new transaction opens. + * + * @data_version: incremented every time the contents of a file could + * have changed. It is exposed via an ioctl and is then provided as an + * argument to data functions to protect racing modification. + * + * @online_blocks: The number of fixed 4k blocks currently allocated and + * storing data in the volume. + * + * @offline_blocks: The number of fixed 4k blocks that could be made + * online by staging. + * + * XXX + * - otime? + * - compat flags? + * - version? + * - generation? + * - be more careful with rdev? + */ +struct scoutfs_inode { + __le64 size; + __le64 meta_seq; + __le64 data_seq; + __le64 data_version; + __le64 online_blocks; + __le64 offline_blocks; + __le64 next_readdir_pos; + __le64 next_xattr_id; + __le32 nlink; + __le32 uid; + __le32 gid; + __le32 mode; + __le32 rdev; + __le32 flags; + struct scoutfs_timespec atime; + struct scoutfs_timespec ctime; + struct scoutfs_timespec mtime; +}; + +#define SCOUTFS_INO_FLAG_TRUNCATE 0x1 + +#define SCOUTFS_ROOT_INO 1 + +/* like the block size, a reasonable min PATH_MAX across platforms */ +#define SCOUTFS_SYMLINK_MAX_SIZE 4096 + +/* + * Dirents are stored in multiple places to isolate contention when + * performing different operations: hashed by name for creation and + * lookup, at incrementing positions for readdir and resolving inodes to + * paths. Each entry has all the metadata needed to reference all the + * items (so an entry cached by lookup can be used to unlink all the + * items). + */ +struct scoutfs_dirent { + __le64 ino; + __le64 hash; + __le64 pos; + __u8 type; + __u8 __pad[7]; + __u8 name[0]; +}; + +#define SCOUTFS_NAME_LEN 255 + +/* S32_MAX avoids the (int) sign bit and might avoid sloppy bugs */ +#define SCOUTFS_LINK_MAX S32_MAX + +/* entries begin after . and .. */ +#define SCOUTFS_DIRENT_FIRST_POS 2 +/* getdents returns next pos with an entry, no entry at (f_pos)~0 */ +#define SCOUTFS_DIRENT_LAST_POS (U64_MAX - 1) + +enum scoutfs_dentry_type { + SCOUTFS_DT_FIFO = 0, + SCOUTFS_DT_CHR, + SCOUTFS_DT_DIR, + SCOUTFS_DT_BLK, + SCOUTFS_DT_REG, + SCOUTFS_DT_LNK, + SCOUTFS_DT_SOCK, + SCOUTFS_DT_WHT, +}; + + +#define SCOUTFS_XATTR_MAX_NAME_LEN 255 +#define SCOUTFS_XATTR_MAX_VAL_LEN 65535 +#define SCOUTFS_XATTR_MAX_PART_SIZE SCOUTFS_MAX_VAL_SIZE + +#define SCOUTFS_XATTR_NR_PARTS(name_len, val_len) \ + DIV_ROUND_UP(sizeof(struct scoutfs_xattr) + name_len + val_len, \ + (unsigned int)SCOUTFS_XATTR_MAX_PART_SIZE) + +#define SCOUTFS_LOCK_INODE_GROUP_NR 1024 +#define SCOUTFS_LOCK_INODE_GROUP_MASK (SCOUTFS_LOCK_INODE_GROUP_NR - 1) +#define SCOUTFS_LOCK_SEQ_GROUP_MASK ((1ULL << 10) - 1) + +/* + * messages over the wire. + */ + +/* + * Greetings verify identity of communicating nodes. The sender sends + * their credentials and the receiver verifies them. + * + * @server_term: The raft term that elected the server. Initially 0 + * from the client, sent by the server, then sent by the client as it + * tries to reconnect. Used to identify a client reconnecting to both + * the same serer after receiving a greeting response and to a new + * server after failover. + * + * @unmount_barrier: Incremented every time the remaining majority of + * quorum members all agree to leave. The server tells a quorum member + * the value that it's connecting under so that if the client sees the + * value increase in the super block then it knows that the server has + * processed its farewell and can safely unmount. + * + * @rid: The client's random id that was generated once as the mount + * started up. This identifies a specific remote mount across + * connections and servers. It's set to the client's rid in both the + * request and response for consistency. + */ +struct scoutfs_net_greeting { + __le64 fsid; + __le64 format_hash; + __le64 server_term; + __le64 unmount_barrier; + __le64 rid; + __le64 flags; +}; + +#define SCOUTFS_NET_GREETING_FLAG_FAREWELL (1 << 0) +#define SCOUTFS_NET_GREETING_FLAG_VOTER (1 << 1) +#define SCOUTFS_NET_GREETING_FLAG_INVALID (~(__u64)0 << 2) + +/* + * This header precedes and describes all network messages sent over + * sockets. + * + * @seq: A sequence number that is increased for each message queued for + * send on the sender. The sender will never reorder messages in the + * send queue so this will always increase in recv on the receiver. The + * receiver can use this to drop messages that arrived twice after being + * resent across a newly connected socket for a given connection. + * + * @recv_seq: The sequence number of the last received message. The + * receiver is sending this to the sender in every message. The sender + * uses them to drop responses which have been delivered. + * + * @id: An increasing identifier that is set in each request. Responses + * specify the request that they're responding to. + * + * Error is only set to a translated errno and will only be found in + * response messages. + */ +struct scoutfs_net_header { + __le64 clock_sync_id; + __le64 seq; + __le64 recv_seq; + __le64 id; + __le16 data_len; + __u8 cmd; + __u8 flags; + __u8 error; + __u8 __pad[3]; + __u8 data[0]; +}; + +#define SCOUTFS_NET_FLAG_RESPONSE (1 << 0) +#define SCOUTFS_NET_FLAGS_UNKNOWN (U8_MAX << 1) + +enum scoutfs_net_cmd { + SCOUTFS_NET_CMD_GREETING = 0, + SCOUTFS_NET_CMD_ALLOC_INODES, + SCOUTFS_NET_CMD_GET_LOG_TREES, + SCOUTFS_NET_CMD_COMMIT_LOG_TREES, + SCOUTFS_NET_CMD_GET_ROOTS, + SCOUTFS_NET_CMD_ADVANCE_SEQ, + SCOUTFS_NET_CMD_GET_LAST_SEQ, + SCOUTFS_NET_CMD_LOCK, + SCOUTFS_NET_CMD_LOCK_RECOVER, + SCOUTFS_NET_CMD_SRCH_GET_COMPACT, + SCOUTFS_NET_CMD_SRCH_COMMIT_COMPACT, + SCOUTFS_NET_CMD_FAREWELL, + SCOUTFS_NET_CMD_UNKNOWN, +}; + +/* + * Define a macro to evaluate another macro for each of the errnos we + * translate over the wire. This lets us keep our enum in sync with the + * mapping arrays to and from host errnos. + */ +#define EXPAND_EACH_NET_ERRNO \ + EXPAND_NET_ERRNO(ENOENT) \ + EXPAND_NET_ERRNO(ENOMEM) \ + EXPAND_NET_ERRNO(EIO) \ + EXPAND_NET_ERRNO(ENOSPC) \ + EXPAND_NET_ERRNO(EINVAL) + +#undef EXPAND_NET_ERRNO +#define EXPAND_NET_ERRNO(which) SCOUTFS_NET_ERR_##which, +enum scoutfs_net_errors { + SCOUTFS_NET_ERR_NONE = 0, + EXPAND_EACH_NET_ERRNO + SCOUTFS_NET_ERR_UNKNOWN, +}; + +/* arbitrarily chosen to be safely less than mss and allow 1k with header */ +#define SCOUTFS_NET_MAX_DATA_LEN 1100 + +/* + * When there's no more free inodes this will be sent with ino = ~0 and + * nr = 0. + */ +struct scoutfs_net_inode_alloc { + __le64 ino; + __le64 nr; +}; + +struct scoutfs_net_roots { + struct scoutfs_btree_root fs_root; + struct scoutfs_btree_root logs_root; + struct scoutfs_btree_root srch_root; +}; + +struct scoutfs_net_lock { + struct scoutfs_key key; + __le64 write_version; + __u8 old_mode; + __u8 new_mode; + __u8 __pad[6]; +}; + +struct scoutfs_net_lock_grant_response { + struct scoutfs_net_lock nl; + struct scoutfs_net_roots roots; +}; + +struct scoutfs_net_lock_recover { + __le16 nr; + __u8 __pad[6]; + struct scoutfs_net_lock locks[0]; +}; + +#define SCOUTFS_NET_LOCK_MAX_RECOVER_NR \ + ((SCOUTFS_NET_MAX_DATA_LEN - sizeof(struct scoutfs_net_lock_recover)) /\ + sizeof(struct scoutfs_net_lock)) + +/* some enums for tracing */ +enum scoutfs_lock_trace { + SLT_CLIENT, + SLT_SERVER, + SLT_GRANT, + SLT_INVALIDATE, + SLT_REQUEST, + SLT_RESPONSE, +}; + +/* + * Read and write locks operate as you'd expect. Multiple readers can + * hold read locks while writers are excluded. A single writer can hold + * a write lock which excludes other readers and writers. Writers can + * read while holding a write lock. + * + * Multiple writers can hold write only locks but they can not read, + * they can only generate dirty items. It's used when the system has + * other means of knowing that it's safe to overwrite items. + * + * The null mode provides no access and is used to destroy locks. + */ +enum scoutfs_lock_mode { + SCOUTFS_LOCK_NULL = 0, + SCOUTFS_LOCK_READ, + SCOUTFS_LOCK_WRITE, + SCOUTFS_LOCK_WRITE_ONLY, + SCOUTFS_LOCK_INVALID, +}; + +/* + * Scoutfs file handle structure - this can be copied out to userspace + * via open by handle or put on the wire from NFS. + */ +struct scoutfs_fid { + __le64 ino; + __le64 parent_ino; +}; + +#define FILEID_SCOUTFS 0x81 +#define FILEID_SCOUTFS_WITH_PARENT 0x82 + +/* + * Identifiers for sources of corruption that can generate messages. + */ +enum scoutfs_corruption_sources { + SC_DIRENT_NAME_LEN = 0, + SC_DIRENT_BACKREF_NAME_LEN, + SC_DIRENT_READDIR_NAME_LEN, + SC_SYMLINK_INODE_SIZE, + SC_SYMLINK_MISSING_ITEM, + SC_SYMLINK_NOT_NULL_TERM, + SC_BTREE_BLOCK_LEVEL, + SC_BTREE_NO_CHILD_REF, + SC_INODE_BLOCK_COUNTS, + SC_NR_SOURCES, +}; + +#define SC_NR_LONGS DIV_ROUND_UP(SC_NR_SOURCES, BITS_PER_LONG) + +#endif diff --git a/kmod/src/hash.h b/kmod/src/hash.h new file mode 100644 index 00000000..cb50b99c --- /dev/null +++ b/kmod/src/hash.h @@ -0,0 +1,49 @@ +#ifndef _SCOUTFS_HASH_H_ +#define _SCOUTFS_HASH_H_ + +/* + * We're using FNV1a for now. It's fine. Ish. + * + * The longer term plan is xxh3 but it looks like it'll take just a bit + * more time to be declared stable and then it needs to be ported to the + * kernel. + * + * - https://fastcompression.blogspot.com/2019/03/presenting-xxh3.html + * - https://github.com/Cyan4973/xxHash/releases/tag/v0.7.4 + */ + +static inline u32 fnv1a32(const void *data, unsigned int len) +{ + u32 hash = 0x811c9dc5; + + while (len--) { + hash ^= *(u8 *)(data++); + hash *= 0x01000193; + } + + return hash; +} + +static inline u64 fnv1a64(const void *data, unsigned int len) +{ + u64 hash = 0xcbf29ce484222325ULL; + + while (len--) { + hash ^= *(u8 *)(data++); + hash *= 0x100000001b3ULL; + } + + return hash; +} + +static inline u32 scoutfs_hash32(const void *data, unsigned int len) +{ + return fnv1a32(data, len); +} + +static inline u64 scoutfs_hash64(const void *data, unsigned int len) +{ + return fnv1a64(data, len); +} + +#endif diff --git a/kmod/src/inode.c b/kmod/src/inode.c new file mode 100644 index 00000000..63aa70b9 --- /dev/null +++ b/kmod/src/inode.c @@ -0,0 +1,1761 @@ +/* + * Copyright (C) 2015 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "format.h" +#include "super.h" +#include "key.h" +#include "inode.h" +#include "dir.h" +#include "data.h" +#include "scoutfs_trace.h" +#include "xattr.h" +#include "trans.h" +#include "msg.h" +#include "item.h" +#include "client.h" +#include "cmp.h" + +/* + * XXX + * - worry about i_ino trunctation, not sure if we do anything + * - use inode item value lengths for forward/back compat + */ + +/* + * XXX before committing: + * - describe all this better + * - describe data locking size problems + */ + +struct inode_allocator { + spinlock_t lock; + u64 ino; + u64 nr; +}; + +struct inode_sb_info { + spinlock_t writeback_lock; + struct rb_root writeback_inodes; + struct inode_allocator dir_ino_alloc; + struct inode_allocator ino_alloc; +}; + +#define DECLARE_INODE_SB_INFO(sb, name) \ + struct inode_sb_info *name = SCOUTFS_SB(sb)->inode_sb_info + +static struct kmem_cache *scoutfs_inode_cachep; + +/* + * This is called once before all the allocations and frees of a inode + * object within a slab. It's for inode fields that don't need to be + * initialized for a given instance of an inode. + */ +static void scoutfs_inode_ctor(void *obj) +{ + struct scoutfs_inode_info *ci = obj; + + mutex_init(&ci->item_mutex); + seqcount_init(&ci->seqcount); + ci->staging = false; + scoutfs_per_task_init(&ci->pt_data_lock); + atomic64_set(&ci->data_waitq.changed, 0); + init_waitqueue_head(&ci->data_waitq.waitq); + init_rwsem(&ci->xattr_rwsem); + RB_CLEAR_NODE(&ci->writeback_node); + + inode_init_once(&ci->inode); +} + +struct inode *scoutfs_alloc_inode(struct super_block *sb) +{ + struct scoutfs_inode_info *ci; + + ci = kmem_cache_alloc(scoutfs_inode_cachep, GFP_NOFS); + if (!ci) + return NULL; + + return &ci->inode; +} + +static void scoutfs_i_callback(struct rcu_head *head) +{ + struct inode *inode = container_of(head, struct inode, i_rcu); + + trace_scoutfs_i_callback(inode); + kmem_cache_free(scoutfs_inode_cachep, SCOUTFS_I(inode)); +} + +static void insert_writeback_inode(struct inode_sb_info *inf, + struct scoutfs_inode_info *ins) +{ + struct rb_root *root = &inf->writeback_inodes; + struct rb_node **node = &root->rb_node; + struct rb_node *parent = NULL; + struct scoutfs_inode_info *si; + + while (*node) { + parent = *node; + si = container_of(*node, struct scoutfs_inode_info, + writeback_node); + + if (ins->ino < si->ino) + node = &(*node)->rb_left; + else if (ins->ino > si->ino) + node = &(*node)->rb_right; + else + BUG(); + } + + rb_link_node(&ins->writeback_node, parent, node); + rb_insert_color(&ins->writeback_node, root); +} + +static void remove_writeback_inode(struct inode_sb_info *inf, + struct scoutfs_inode_info *si) +{ + if (!RB_EMPTY_NODE(&si->writeback_node)) { + rb_erase(&si->writeback_node, &inf->writeback_inodes); + RB_CLEAR_NODE(&si->writeback_node); + } +} + +void scoutfs_destroy_inode(struct inode *inode) +{ + DECLARE_INODE_SB_INFO(inode->i_sb, inf); + + spin_lock(&inf->writeback_lock); + remove_writeback_inode(inf, SCOUTFS_I(inode)); + spin_unlock(&inf->writeback_lock); + + call_rcu(&inode->i_rcu, scoutfs_i_callback); +} + +static const struct inode_operations scoutfs_file_iops = { + .getattr = scoutfs_getattr, + .setattr = scoutfs_setattr, + .setxattr = scoutfs_setxattr, + .getxattr = scoutfs_getxattr, + .listxattr = scoutfs_listxattr, + .removexattr = scoutfs_removexattr, + .fiemap = scoutfs_data_fiemap, +}; + +static const struct inode_operations scoutfs_special_iops = { + .getattr = scoutfs_getattr, + .setattr = scoutfs_setattr, + .setxattr = scoutfs_setxattr, + .getxattr = scoutfs_getxattr, + .listxattr = scoutfs_listxattr, + .removexattr = scoutfs_removexattr, +}; + +/* + * Called once new inode allocation or inode reading has initialized + * enough of the inode for us to set the ops based on the mode. + */ +static void set_inode_ops(struct inode *inode) +{ + switch (inode->i_mode & S_IFMT) { + case S_IFREG: + inode->i_mapping->a_ops = &scoutfs_file_aops; + inode->i_op = &scoutfs_file_iops; + inode->i_fop = &scoutfs_file_fops; + break; + case S_IFDIR: + inode->i_op = &scoutfs_dir_iops; + inode->i_fop = &scoutfs_dir_fops; + break; + case S_IFLNK: + inode->i_op = &scoutfs_symlink_iops; + break; + default: + inode->i_op = &scoutfs_special_iops; + init_special_inode(inode, inode->i_mode, inode->i_rdev); + break; + } + + /* ephemeral data items avoid kmap for pointers to page contents */ + mapping_set_gfp_mask(inode->i_mapping, GFP_USER); +} + +/* + * The caller has ensured that the fields in the incoming scoutfs inode + * reflect both the inode item and the inode index items. This happens + * when reading, refreshing, or updating the inodes. We set the inode + * info fields to match so that next time we try to update the inode we + * can tell which fields have changed. + */ +static void set_item_info(struct scoutfs_inode_info *si, + struct scoutfs_inode *sinode) +{ + BUG_ON(!mutex_is_locked(&si->item_mutex)); + + memset(si->item_majors, 0, sizeof(si->item_majors)); + memset(si->item_minors, 0, sizeof(si->item_minors)); + + si->have_item = true; + si->item_majors[SCOUTFS_INODE_INDEX_META_SEQ_TYPE] = + le64_to_cpu(sinode->meta_seq); + si->item_majors[SCOUTFS_INODE_INDEX_DATA_SEQ_TYPE] = + le64_to_cpu(sinode->data_seq); +} + +static void load_inode(struct inode *inode, struct scoutfs_inode *cinode) +{ + struct scoutfs_inode_info *ci = SCOUTFS_I(inode); + + i_size_write(inode, le64_to_cpu(cinode->size)); + set_nlink(inode, le32_to_cpu(cinode->nlink)); + i_uid_write(inode, le32_to_cpu(cinode->uid)); + i_gid_write(inode, le32_to_cpu(cinode->gid)); + inode->i_mode = le32_to_cpu(cinode->mode); + inode->i_rdev = le32_to_cpu(cinode->rdev); + inode->i_atime.tv_sec = le64_to_cpu(cinode->atime.sec); + inode->i_atime.tv_nsec = le32_to_cpu(cinode->atime.nsec); + inode->i_mtime.tv_sec = le64_to_cpu(cinode->mtime.sec); + inode->i_mtime.tv_nsec = le32_to_cpu(cinode->mtime.nsec); + inode->i_ctime.tv_sec = le64_to_cpu(cinode->ctime.sec); + inode->i_ctime.tv_nsec = le32_to_cpu(cinode->ctime.nsec); + + ci->meta_seq = le64_to_cpu(cinode->meta_seq); + ci->data_seq = le64_to_cpu(cinode->data_seq); + ci->data_version = le64_to_cpu(cinode->data_version); + ci->online_blocks = le64_to_cpu(cinode->online_blocks); + ci->offline_blocks = le64_to_cpu(cinode->offline_blocks); + ci->next_readdir_pos = le64_to_cpu(cinode->next_readdir_pos); + ci->next_xattr_id = le64_to_cpu(cinode->next_xattr_id); + ci->flags = le32_to_cpu(cinode->flags); + + /* + * i_blocks is initialized from online and offline and is then + * maintained as blocks come and go. + */ + inode->i_blocks = (ci->online_blocks + ci->offline_blocks) + << SCOUTFS_BLOCK_SM_SECTOR_SHIFT; + + set_item_info(ci, cinode); +} + +static void init_inode_key(struct scoutfs_key *key, u64 ino) +{ + *key = (struct scoutfs_key) { + .sk_zone = SCOUTFS_FS_ZONE, + .ski_ino = cpu_to_le64(ino), + .sk_type = SCOUTFS_INODE_TYPE, + }; +} + +/* + * Refresh the vfs inode fields if the lock indicates that the current + * contents could be stale. + * + * This can be racing with many lock holders of an inode. A bunch of + * readers can be checking to refresh while one of them is refreshing. + * + * The vfs inode field updates can't be racing with valid readers of the + * fields because they should have already had a locked refreshed inode + * to be dereferencing its contents. + */ +int scoutfs_inode_refresh(struct inode *inode, struct scoutfs_lock *lock, + int flags) +{ + struct scoutfs_inode_info *si = SCOUTFS_I(inode); + struct super_block *sb = inode->i_sb; + struct scoutfs_key key; + struct scoutfs_inode sinode; + const u64 refresh_gen = lock->refresh_gen; + int ret; + + /* + * Lock refresh gens are supposed to strictly increase. Inodes + * having a greater gen means memory corruption or + * lifetime/logic bugs that could stop the inode from refreshing + * and expose stale data. + */ + BUG_ON(atomic64_read(&si->last_refreshed) > refresh_gen); + + if (atomic64_read(&si->last_refreshed) == refresh_gen) + return 0; + + init_inode_key(&key, scoutfs_ino(inode)); + + mutex_lock(&si->item_mutex); + if (atomic64_read(&si->last_refreshed) < refresh_gen) { + ret = scoutfs_item_lookup_exact(sb, &key, &sinode, + sizeof(sinode), lock); + if (ret == 0) { + load_inode(inode, &sinode); + atomic64_set(&si->last_refreshed, refresh_gen); + } + } else { + ret = 0; + } + mutex_unlock(&si->item_mutex); + + return ret; +} + +int scoutfs_getattr(struct vfsmount *mnt, struct dentry *dentry, + struct kstat *stat) +{ + struct inode *inode = dentry->d_inode; + struct super_block *sb = inode->i_sb; + struct scoutfs_lock *lock = NULL; + int ret; + + ret = scoutfs_lock_inode(sb, SCOUTFS_LOCK_READ, + SCOUTFS_LKF_REFRESH_INODE, inode, &lock); + if (ret == 0) { + generic_fillattr(inode, stat); + scoutfs_unlock(sb, lock, SCOUTFS_LOCK_READ); + } + return ret; +} + +static int set_inode_size(struct inode *inode, struct scoutfs_lock *lock, + u64 new_size, bool truncate) +{ + struct scoutfs_inode_info *ci = SCOUTFS_I(inode); + struct super_block *sb = inode->i_sb; + LIST_HEAD(ind_locks); + int ret; + + if (!S_ISREG(inode->i_mode)) + return 0; + + ret = scoutfs_inode_index_lock_hold(inode, &ind_locks, true, + SIC_DIRTY_INODE()); + if (ret) + return ret; + + if (new_size != i_size_read(inode)) + scoutfs_inode_inc_data_version(inode); + + truncate_setsize(inode, new_size); + inode->i_ctime = inode->i_mtime = CURRENT_TIME; + if (truncate) + ci->flags |= SCOUTFS_INO_FLAG_TRUNCATE; + scoutfs_inode_set_data_seq(inode); + scoutfs_update_inode_item(inode, lock, &ind_locks); + + scoutfs_release_trans(sb); + scoutfs_inode_index_unlock(sb, &ind_locks); + + return ret; +} + +static int clear_truncate_flag(struct inode *inode, struct scoutfs_lock *lock) +{ + struct scoutfs_inode_info *ci = SCOUTFS_I(inode); + struct super_block *sb = inode->i_sb; + LIST_HEAD(ind_locks); + int ret; + + ret = scoutfs_inode_index_lock_hold(inode, &ind_locks, false, + SIC_DIRTY_INODE()); + if (ret) + return ret; + + ci->flags &= ~SCOUTFS_INO_FLAG_TRUNCATE; + scoutfs_update_inode_item(inode, lock, &ind_locks); + + scoutfs_release_trans(sb); + scoutfs_inode_index_unlock(sb, &ind_locks); + + return ret; +} + +int scoutfs_complete_truncate(struct inode *inode, struct scoutfs_lock *lock) +{ + struct scoutfs_inode_info *ci = SCOUTFS_I(inode); + u64 start; + int ret, err; + + trace_scoutfs_complete_truncate(inode, ci->flags); + + if (!(ci->flags & SCOUTFS_INO_FLAG_TRUNCATE)) + return 0; + + start = (i_size_read(inode) + SCOUTFS_BLOCK_SM_SIZE - 1) >> + SCOUTFS_BLOCK_SM_SHIFT; + ret = scoutfs_data_truncate_items(inode->i_sb, inode, + scoutfs_ino(inode), start, ~0ULL, + false, lock); + err = clear_truncate_flag(inode, lock); + + return ret ? ret : err; +} + +/* + * If we're changing the file size than the contents of the file are + * changing and we increment the data_version. This would prevent + * staging because the data_version is per-inode today, not per-extent. + * So if there are any offline extents within the new size then we need + * to stage them before we truncate. And this is called with the + * i_mutex held which would prevent staging so we release it and + * re-acquire it. Ideally we'd fix this so that we can acquire the lock + * instead of the caller. + */ +int scoutfs_setattr(struct dentry *dentry, struct iattr *attr) +{ + struct inode *inode = dentry->d_inode; + struct super_block *sb = inode->i_sb; + struct scoutfs_lock *lock = NULL; + DECLARE_DATA_WAIT(dw); + LIST_HEAD(ind_locks); + bool truncate = false; + u64 attr_size; + int ret; + + trace_scoutfs_setattr(dentry, attr); + +retry: + ret = scoutfs_lock_inode(sb, SCOUTFS_LOCK_WRITE, + SCOUTFS_LKF_REFRESH_INODE, inode, &lock); + if (ret) + return ret; + + ret = inode_change_ok(inode, attr); + if (ret) + goto out; + + attr_size = (attr->ia_valid & ATTR_SIZE) ? attr->ia_size : + i_size_read(inode); + + if (S_ISREG(inode->i_mode) && attr->ia_valid & ATTR_SIZE) { + /* + * Complete any truncates that may have failed while + * in progress + */ + ret = scoutfs_complete_truncate(inode, lock); + if (ret) + goto out; + + /* data_version is per inode, all must be online */ + if (attr_size > 0 && attr_size != i_size_read(inode)) { + ret = scoutfs_data_wait_check(inode, 0, attr_size, + SEF_OFFLINE, + SCOUTFS_IOC_DWO_CHANGE_SIZE, + &dw, lock); + if (ret < 0) + goto out; + if (scoutfs_data_wait_found(&dw)) { + scoutfs_unlock(sb, lock, SCOUTFS_LOCK_WRITE); + + /* XXX callee locks instead? */ + mutex_unlock(&inode->i_mutex); + ret = scoutfs_data_wait(inode, &dw); + mutex_lock(&inode->i_mutex); + + if (ret == 0) + goto retry; + goto out; + } + } + + /* truncating to current size truncates extents past size */ + truncate = i_size_read(inode) >= attr_size; + + ret = set_inode_size(inode, lock, attr_size, truncate); + if (ret) + goto out; + + if (truncate) { + ret = scoutfs_complete_truncate(inode, lock); + if (ret) + goto out; + } + } + + ret = scoutfs_inode_index_lock_hold(inode, &ind_locks, false, + SIC_DIRTY_INODE()); + if (ret) + goto out; + + setattr_copy(inode, attr); + scoutfs_update_inode_item(inode, lock, &ind_locks); + + scoutfs_release_trans(sb); + scoutfs_inode_index_unlock(sb, &ind_locks); +out: + scoutfs_unlock(sb, lock, SCOUTFS_LOCK_WRITE); + return ret; +} + +/* + * Set a given seq to the current trans seq if it differs. The caller + * holds locks and a transaction which prevents the transaction from + * committing and refreshing the seq. + */ +static void set_trans_seq(struct inode *inode, u64 *seq) +{ + struct scoutfs_inode_info *si = SCOUTFS_I(inode); + struct super_block *sb = inode->i_sb; + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + + if (*seq != sbi->trans_seq) { + preempt_disable(); + write_seqcount_begin(&si->seqcount); + *seq = sbi->trans_seq; + write_seqcount_end(&si->seqcount); + preempt_enable(); + } +} + +void scoutfs_inode_set_meta_seq(struct inode *inode) +{ + struct scoutfs_inode_info *si = SCOUTFS_I(inode); + + set_trans_seq(inode, &si->meta_seq); +} + +void scoutfs_inode_set_data_seq(struct inode *inode) +{ + struct scoutfs_inode_info *si = SCOUTFS_I(inode); + + set_trans_seq(inode, &si->data_seq); +} + +void scoutfs_inode_inc_data_version(struct inode *inode) +{ + struct scoutfs_inode_info *si = SCOUTFS_I(inode); + + preempt_disable(); + write_seqcount_begin(&si->seqcount); + si->data_version++; + write_seqcount_end(&si->seqcount); + preempt_enable(); +} + +void scoutfs_inode_set_data_version(struct inode *inode, u64 data_version) +{ + struct scoutfs_inode_info *si = SCOUTFS_I(inode); + + preempt_disable(); + write_seqcount_begin(&si->seqcount); + si->data_version = data_version; + write_seqcount_end(&si->seqcount); + preempt_enable(); +} + +void scoutfs_inode_add_onoff(struct inode *inode, s64 on, s64 off) +{ + struct scoutfs_inode_info *si; + + if (inode && (on || off)) { + si = SCOUTFS_I(inode); + preempt_disable(); + write_seqcount_begin(&si->seqcount); + + /* inode and extents out of sync, bad callers */ + if (((s64)si->online_blocks + on < 0) || + ((s64)si->offline_blocks + off < 0)) { + scoutfs_corruption(inode->i_sb, SC_INODE_BLOCK_COUNTS, + corrupt_inode_block_counts, + "ino %llu size %llu online %llu + %lld offline %llu + %lld", + scoutfs_ino(inode), i_size_read(inode), + si->online_blocks, on, si->offline_blocks, off); + } + + si->online_blocks += on; + si->offline_blocks += off; + /* XXX not sure if this is right */ + inode->i_blocks += (on + off) * SCOUTFS_BLOCK_SM_SECTORS; + + trace_scoutfs_online_offline_blocks(inode, on, off, + si->online_blocks, + si->offline_blocks); + + write_seqcount_end(&si->seqcount); + preempt_enable(); + } + + /* any time offline extents decreased we try and wake waiters */ + if (inode && off < 0) + scoutfs_data_wait_changed(inode); +} + +static u64 read_seqcount_u64(struct inode *inode, u64 *val) +{ + struct scoutfs_inode_info *si = SCOUTFS_I(inode); + unsigned int seq; + u64 v; + + do { + seq = read_seqcount_begin(&si->seqcount); + v = *val; + } while (read_seqcount_retry(&si->seqcount, seq)); + + return v; +} + +u64 scoutfs_inode_meta_seq(struct inode *inode) +{ + struct scoutfs_inode_info *si = SCOUTFS_I(inode); + + return read_seqcount_u64(inode, &si->meta_seq); +} + +u64 scoutfs_inode_data_seq(struct inode *inode) +{ + struct scoutfs_inode_info *si = SCOUTFS_I(inode); + + return read_seqcount_u64(inode, &si->data_seq); +} + +u64 scoutfs_inode_data_version(struct inode *inode) +{ + struct scoutfs_inode_info *si = SCOUTFS_I(inode); + + return read_seqcount_u64(inode, &si->data_version); +} + +void scoutfs_inode_get_onoff(struct inode *inode, s64 *on, s64 *off) +{ + struct scoutfs_inode_info *si = SCOUTFS_I(inode); + unsigned int seq; + + do { + seq = read_seqcount_begin(&si->seqcount); + *on = SCOUTFS_I(inode)->online_blocks; + *off = SCOUTFS_I(inode)->offline_blocks; + } while (read_seqcount_retry(&si->seqcount, seq)); +} + +static int scoutfs_iget_test(struct inode *inode, void *arg) +{ + struct scoutfs_inode_info *ci = SCOUTFS_I(inode); + u64 *ino = arg; + + return ci->ino == *ino; +} + +static int scoutfs_iget_set(struct inode *inode, void *arg) +{ + struct scoutfs_inode_info *ci = SCOUTFS_I(inode); + u64 *ino = arg; + + inode->i_ino = *ino; + ci->ino = *ino; + + return 0; +} + +struct inode *scoutfs_ilookup(struct super_block *sb, u64 ino) +{ + return ilookup5(sb, ino, scoutfs_iget_test, &ino); +} + +struct inode *scoutfs_iget(struct super_block *sb, u64 ino) +{ + struct scoutfs_lock *lock = NULL; + struct scoutfs_inode_info *si; + struct inode *inode; + int ret; + + ret = scoutfs_lock_ino(sb, SCOUTFS_LOCK_READ, 0, ino, &lock); + if (ret) + return ERR_PTR(ret); + + inode = iget5_locked(sb, ino, scoutfs_iget_test, scoutfs_iget_set, + &ino); + if (!inode) { + inode = ERR_PTR(-ENOMEM); + goto out; + } + + if (inode->i_state & I_NEW) { + /* XXX ensure refresh, instead clear in drop_inode? */ + si = SCOUTFS_I(inode); + atomic64_set(&si->last_refreshed, 0); + + ret = scoutfs_inode_refresh(inode, lock, 0); + if (ret) { + iget_failed(inode); + inode = ERR_PTR(ret); + } else { + set_inode_ops(inode); + unlock_new_inode(inode); + } + } + +out: + scoutfs_unlock(sb, lock, SCOUTFS_LOCK_READ); + return inode; +} + +static void store_inode(struct scoutfs_inode *cinode, struct inode *inode) +{ + struct scoutfs_inode_info *ci = SCOUTFS_I(inode); + u64 online_blocks; + u64 offline_blocks; + + scoutfs_inode_get_onoff(inode, &online_blocks, &offline_blocks); + + cinode->size = cpu_to_le64(i_size_read(inode)); + cinode->nlink = cpu_to_le32(inode->i_nlink); + cinode->uid = cpu_to_le32(i_uid_read(inode)); + cinode->gid = cpu_to_le32(i_gid_read(inode)); + cinode->mode = cpu_to_le32(inode->i_mode); + cinode->rdev = cpu_to_le32(inode->i_rdev); + cinode->atime.sec = cpu_to_le64(inode->i_atime.tv_sec); + cinode->atime.nsec = cpu_to_le32(inode->i_atime.tv_nsec); + memset(cinode->atime.__pad, 0, sizeof(cinode->atime.__pad)); + cinode->ctime.sec = cpu_to_le64(inode->i_ctime.tv_sec); + cinode->ctime.nsec = cpu_to_le32(inode->i_ctime.tv_nsec); + memset(cinode->ctime.__pad, 0, sizeof(cinode->ctime.__pad)); + cinode->mtime.sec = cpu_to_le64(inode->i_mtime.tv_sec); + cinode->mtime.nsec = cpu_to_le32(inode->i_mtime.tv_nsec); + memset(cinode->mtime.__pad, 0, sizeof(cinode->mtime.__pad)); + + cinode->meta_seq = cpu_to_le64(scoutfs_inode_meta_seq(inode)); + cinode->data_seq = cpu_to_le64(scoutfs_inode_data_seq(inode)); + cinode->data_version = cpu_to_le64(scoutfs_inode_data_version(inode)); + cinode->online_blocks = cpu_to_le64(online_blocks); + cinode->offline_blocks = cpu_to_le64(offline_blocks); + cinode->next_readdir_pos = cpu_to_le64(ci->next_readdir_pos); + cinode->next_xattr_id = cpu_to_le64(ci->next_xattr_id); + cinode->flags = cpu_to_le32(ci->flags); +} + +/* + * Create a pinned dirty inode item so that we can later update the + * inode item without risking failure. We often wouldn't want to have + * to unwind inode modifcations (perhaps by shared vfs code!) if our + * item update failed. This is our chance to return errors for enospc + * for lack of space for new logged dirty inode items. + * + * This dirty inode item will be found by lookups in the interim so we + * have to update it now with the current inode contents. + * + * Callers don't delete these dirty items on errors. They're still + * valid and will be merged with the current item eventually. + * + * The caller has to prevent sync between dirtying and updating the + * inodes. + * + * XXX this will have to do something about variable length inodes + */ +int scoutfs_dirty_inode_item(struct inode *inode, struct scoutfs_lock *lock) +{ + struct super_block *sb = inode->i_sb; + struct scoutfs_inode sinode; + struct scoutfs_key key; + int ret; + + store_inode(&sinode, inode); + + init_inode_key(&key, scoutfs_ino(inode)); + + ret = scoutfs_item_update(sb, &key, &sinode, sizeof(sinode), lock); + if (!ret) + trace_scoutfs_dirty_inode(inode); + return ret; +} + +struct index_lock { + struct list_head head; + struct scoutfs_lock *lock; + u8 type; + u64 major; + u32 minor; + u64 ino; +}; + +static bool will_del_index(struct scoutfs_inode_info *si, + u8 type, u64 major, u32 minor) +{ + return si && si->have_item && + (si->item_majors[type] != major || + si->item_minors[type] != minor); +} + +static bool will_ins_index(struct scoutfs_inode_info *si, + u8 type, u64 major, u32 minor) +{ + return !si || !si->have_item || + (si->item_majors[type] != major || + si->item_minors[type] != minor); +} + +static bool inode_has_index(umode_t mode, u8 type) +{ + switch(type) { + case SCOUTFS_INODE_INDEX_META_SEQ_TYPE: + return true; + case SCOUTFS_INODE_INDEX_DATA_SEQ_TYPE: + return S_ISREG(mode); + default: + return WARN_ON_ONCE(false); + } +} + +static int cmp_index_lock(void *priv, struct list_head *A, struct list_head *B) +{ + struct index_lock *a = list_entry(A, struct index_lock, head); + struct index_lock *b = list_entry(B, struct index_lock, head); + + return ((int)a->type - (int)b->type) ?: + scoutfs_cmp_u64s(a->major, b->major) ?: + scoutfs_cmp_u64s(a->minor, b->minor) ?: + scoutfs_cmp_u64s(a->ino, b->ino); +} + +static void clamp_inode_index(u8 type, u64 *major, u32 *minor, u64 *ino) +{ + struct scoutfs_key start; + + scoutfs_lock_get_index_item_range(type, *major, *ino, &start, NULL); + + *major = le64_to_cpu(start.skii_major); + *minor = 0; + *ino = le64_to_cpu(start.skii_ino); +} + +/* + * Find the lock that covers the given index item. Returns NULL if + * there isn't a lock that covers the item. We know that the list is + * sorted at this point so we can stop once our search value is less + * than a list entry. + */ +static struct scoutfs_lock *find_index_lock(struct list_head *lock_list, + u8 type, u64 major, u32 minor, + u64 ino) +{ + struct index_lock *ind_lock; + struct index_lock needle; + int cmp; + + clamp_inode_index(type, &major, &minor, &ino); + needle.type = type; + needle.major = major; + needle.minor = minor; + needle.ino = ino; + + list_for_each_entry(ind_lock, lock_list, head) { + cmp = cmp_index_lock(NULL, &needle.head, &ind_lock->head); + if (cmp == 0) + return ind_lock->lock; + if (cmp < 0) + break; + } + + return NULL; +} + +void scoutfs_inode_init_index_key(struct scoutfs_key *key, u8 type, u64 major, + u32 minor, u64 ino) +{ + *key = (struct scoutfs_key) { + .sk_zone = SCOUTFS_INODE_INDEX_ZONE, + .sk_type = type, + .skii_major = cpu_to_le64(major), + .skii_ino = cpu_to_le64(ino), + }; +} + +/* + * The inode info reflects the current inode index items. Create or delete + * index items to bring the index in line with the caller's item. The list + * should contain locks that cover any item modifications that are made. + */ +static int update_index_items(struct super_block *sb, + struct scoutfs_inode_info *si, u64 ino, u8 type, + u64 major, u32 minor, + struct list_head *lock_list) +{ + struct scoutfs_lock *ins_lock; + struct scoutfs_lock *del_lock; + struct scoutfs_key ins; + struct scoutfs_key del; + int ret; + int err; + + if (!will_ins_index(si, type, major, minor)) + return 0; + + trace_scoutfs_create_index_item(sb, type, major, minor, ino); + + scoutfs_inode_init_index_key(&ins, type, major, minor, ino); + + ins_lock = find_index_lock(lock_list, type, major, minor, ino); + ret = scoutfs_item_create_force(sb, &ins, NULL, 0, ins_lock); + if (ret || !will_del_index(si, type, major, minor)) + return ret; + + trace_scoutfs_delete_index_item(sb, type, si->item_majors[type], + si->item_minors[type], ino); + + scoutfs_inode_init_index_key(&del, type, si->item_majors[type], + si->item_minors[type], ino); + + del_lock = find_index_lock(lock_list, type, si->item_majors[type], + si->item_minors[type], ino); + ret = scoutfs_item_delete_force(sb, &del, del_lock); + if (ret) { + err = scoutfs_item_delete(sb, &ins, ins_lock); + BUG_ON(err); + } + + return ret; +} + +static int update_indices(struct super_block *sb, + struct scoutfs_inode_info *si, u64 ino, umode_t mode, + struct scoutfs_inode *sinode, + struct list_head *lock_list) +{ + struct index_update { + u8 type; + u64 major; + u32 minor; + } *upd, upds[] = { + { SCOUTFS_INODE_INDEX_META_SEQ_TYPE, + le64_to_cpu(sinode->meta_seq), 0 }, + { SCOUTFS_INODE_INDEX_DATA_SEQ_TYPE, + le64_to_cpu(sinode->data_seq), 0 }, + }; + int ret; + int i; + + for (i = 0, upd = upds; i < ARRAY_SIZE(upds); i++, upd++) { + if (!inode_has_index(mode, upd->type)) + continue; + + ret = update_index_items(sb, si, ino, upd->type, upd->major, + upd->minor, lock_list); + if (ret) + break; + } + + return ret; +} + +/* + * Every time we modify the inode in memory we copy it to its inode + * item. This lets us write out items without having to track down + * dirty vfs inodes. + * + * The caller makes sure that the item is dirty and pinned so they don't + * have to deal with errors and unwinding after they've modified the vfs + * inode and get here. + * + * Index items that track inode fields are updated here as we update the + * inode item. The caller must have acquired locks on all the index + * items that might change. + */ +void scoutfs_update_inode_item(struct inode *inode, struct scoutfs_lock *lock, + struct list_head *lock_list) +{ + struct scoutfs_inode_info *si = SCOUTFS_I(inode); + struct super_block *sb = inode->i_sb; + const u64 ino = scoutfs_ino(inode); + struct scoutfs_key key; + struct scoutfs_inode sinode; + int ret; + int err; + + mutex_lock(&si->item_mutex); + + /* set the meta version once per trans for any inode updates */ + scoutfs_inode_set_meta_seq(inode); + + /* only race with other inode field stores once */ + store_inode(&sinode, inode); + + ret = update_indices(sb, si, ino, inode->i_mode, &sinode, lock_list); + BUG_ON(ret); + + init_inode_key(&key, ino); + + err = scoutfs_item_update(sb, &key, &sinode, sizeof(sinode), lock); + if (err) { + scoutfs_err(sb, "inode %llu update err %d", ino, err); + BUG_ON(err); + } + + set_item_info(si, &sinode); + trace_scoutfs_update_inode(inode); + + mutex_unlock(&si->item_mutex); +} + +/* + * We map the item to coarse locks here. This reduces the number of + * locks we track and means that when we later try to find the lock that + * covers an item we can deal with the item update changing a little + * while still being covered. It does mean we have to share some logic + * with lock naming. + */ +static int add_index_lock(struct list_head *list, u64 ino, u8 type, u64 major, + u32 minor) +{ + struct index_lock *ind_lock; + + clamp_inode_index(type, &major, &minor, &ino); + + list_for_each_entry(ind_lock, list, head) { + if (ind_lock->type == type && ind_lock->major == major && + ind_lock->minor == minor && ind_lock->ino == ino) { + return 0; + } + } + + ind_lock = kzalloc(sizeof(struct index_lock), GFP_NOFS); + if (!ind_lock) + return -ENOMEM; + + ind_lock->type = type; + ind_lock->major = major; + ind_lock->minor = minor; + ind_lock->ino = ino; + list_add(&ind_lock->head, list); + + return 0; +} + +static int prepare_index_items(struct scoutfs_inode_info *si, + struct list_head *list, u64 ino, umode_t mode, + u8 type, u64 major, u32 minor) +{ + int ret; + + if (will_ins_index(si, type, major, minor)) { + ret = add_index_lock(list, ino, type, major, minor); + if (ret) + return ret; + } + + if (will_del_index(si, type, major, minor)) { + ret = add_index_lock(list, ino, type, si->item_majors[type], + si->item_minors[type]); + if (ret) + return ret; + } + + return 0; +} + +/* + * Return the data seq that we expect to see in the updated inode. The + * caller tells us if they know they're going to update it. If the + * inode doesn't exist it'll also get the current data_seq. + */ +static u64 upd_data_seq(struct scoutfs_sb_info *sbi, + struct scoutfs_inode_info *si, bool set_data_seq) +{ + if (!si || !si->have_item || set_data_seq) + return sbi->trans_seq; + + return si->item_majors[SCOUTFS_INODE_INDEX_DATA_SEQ_TYPE]; +} + +/* + * Prepare locks that will cover the inode index items that will be + * modified when this inode's item is updated during the upcoming + * transaction. + * + * To lock the index items that will be created we need to predict the + * new indexed values. We assume that the meta seq will always be set + * to the current seq. This will usually be a nop in a running + * transaction. The caller tells us what the size will be and whether + * data_seq will also be set to the current transaction. + */ +static int prepare_indices(struct super_block *sb, struct list_head *list, + struct scoutfs_inode_info *si, u64 ino, + umode_t mode, bool set_data_seq) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct index_update { + u8 type; + u64 major; + u32 minor; + } *upd, upds[] = { + { SCOUTFS_INODE_INDEX_META_SEQ_TYPE, sbi->trans_seq, 0}, + { SCOUTFS_INODE_INDEX_DATA_SEQ_TYPE, + upd_data_seq(sbi, si, set_data_seq), 0}, + }; + int ret; + int i; + + for (i = 0, upd = upds; i < ARRAY_SIZE(upds); i++, upd++) { + if (!inode_has_index(mode, upd->type)) + continue; + + ret = prepare_index_items(si, list, ino, mode, + upd->type, upd->major, upd->minor); + if (ret) + break; + } + + return ret; +} + +int scoutfs_inode_index_prepare(struct super_block *sb, struct list_head *list, + struct inode *inode, bool set_data_seq) +{ + struct scoutfs_inode_info *si = SCOUTFS_I(inode); + + return prepare_indices(sb, list, si, scoutfs_ino(inode), + inode->i_mode, set_data_seq); +} + +/* + * This is used to initially create the index items for a newly created + * inode. We don't have a populated vfs inode yet. The existing + * indexed values don't matter because it's 'have_item' is false. It + * will try to create all the appropriate index items. + */ +int scoutfs_inode_index_prepare_ino(struct super_block *sb, + struct list_head *list, u64 ino, + umode_t mode) +{ + return prepare_indices(sb, list, NULL, ino, mode, true); +} + +/* + * Prepare the locks needed to delete all the index items associated + * with the inode. We know the items have to exist and can skip straight + * to adding locks for each of them. + */ +static int prepare_index_deletion(struct super_block *sb, + struct list_head *list, u64 ino, + umode_t mode, struct scoutfs_inode *sinode) +{ + struct index_item { + u8 type; + u64 major; + u32 minor; + } *ind, inds[] = { + { SCOUTFS_INODE_INDEX_META_SEQ_TYPE, + le64_to_cpu(sinode->meta_seq), 0 }, + { SCOUTFS_INODE_INDEX_DATA_SEQ_TYPE, + le64_to_cpu(sinode->data_seq), 0 }, + }; + int ret; + int i; + + for (i = 0, ind = inds; i < ARRAY_SIZE(inds); i++, ind++) { + if (!inode_has_index(mode, ind->type)) + continue; + + ret = add_index_lock(list, ino, ind->type, ind->major, + ind->minor); + if (ret) + break; + } + + return ret; +} + +/* + * Sample the transaction sequence before we start checking it to see if + * indexed meta seq and data seq items will change. + */ +int scoutfs_inode_index_start(struct super_block *sb, u64 *seq) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + + /* XXX this feels racey in a bad way :) */ + *seq = sbi->trans_seq; + return 0; +} + +/* + * Acquire the prepared index locks and hold the transaction. If the + * sequence number changes as we enter the transaction then we need to + * retry so that we can use the new seq to prepare locks. + * + * Returns > 0 if the seq changed and the locks should be retried. + */ +int scoutfs_inode_index_try_lock_hold(struct super_block *sb, + struct list_head *list, u64 seq, + const struct scoutfs_item_count cnt) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct index_lock *ind_lock; + int ret = 0; + + list_sort(NULL, list, cmp_index_lock); + + list_for_each_entry(ind_lock, list, head) { + ret = scoutfs_lock_inode_index(sb, SCOUTFS_LOCK_WRITE_ONLY, + ind_lock->type, ind_lock->major, + ind_lock->ino, &ind_lock->lock); + if (ret) + goto out; + } + + ret = scoutfs_hold_trans(sb, cnt); + if (ret == 0 && seq != sbi->trans_seq) { + scoutfs_release_trans(sb); + ret = 1; + } + +out: + if (ret) + scoutfs_inode_index_unlock(sb, list); + + return ret; +} + +int scoutfs_inode_index_lock_hold(struct inode *inode, struct list_head *list, + bool set_data_seq, + const struct scoutfs_item_count cnt) +{ + struct super_block *sb = inode->i_sb; + int ret; + u64 seq; + + do { + ret = scoutfs_inode_index_start(sb, &seq) ?: + scoutfs_inode_index_prepare(sb, list, inode, + set_data_seq) ?: + scoutfs_inode_index_try_lock_hold(sb, list, seq, cnt); + } while (ret > 0); + + return ret; +} + +/* + * Unlocks and frees all the locks on the list. + */ +void scoutfs_inode_index_unlock(struct super_block *sb, struct list_head *list) +{ + struct index_lock *ind_lock; + struct index_lock *tmp; + + list_for_each_entry_safe(ind_lock, tmp, list, head) { + scoutfs_unlock(sb, ind_lock->lock, SCOUTFS_LOCK_WRITE_ONLY); + list_del_init(&ind_lock->head); + kfree(ind_lock); + } +} + +/* this is called on final inode cleanup so enoent is fine */ +static int remove_index(struct super_block *sb, u64 ino, u8 type, u64 major, + u32 minor, struct list_head *ind_locks) +{ + struct scoutfs_key key; + struct scoutfs_lock *lock; + int ret; + + scoutfs_inode_init_index_key(&key, type, major, minor, ino); + + lock = find_index_lock(ind_locks, type, major, minor, ino); + ret = scoutfs_item_delete_force(sb, &key, lock); + if (ret == -ENOENT) + ret = 0; + return ret; +} + +/* + * Remove all the inode's index items. The caller has ensured that + * there are no more active users of the inode. This can be racing with + * users of the inode index items. Once we can use them we'll get CW + * locks around the index items to invalidate remote caches. Racing + * users of the index items already have to deal with the possibility + * that the inodes returned by the index queries can go out of sync by + * the time they get to it, including being deleted. + */ +static int remove_index_items(struct super_block *sb, u64 ino, + struct scoutfs_inode *sinode, + struct list_head *ind_locks) +{ + umode_t mode = le32_to_cpu(sinode->mode); + int ret; + + ret = remove_index(sb, ino, SCOUTFS_INODE_INDEX_META_SEQ_TYPE, + le64_to_cpu(sinode->meta_seq), 0, ind_locks); + if (ret == 0 && S_ISREG(mode)) + ret = remove_index(sb, ino, SCOUTFS_INODE_INDEX_DATA_SEQ_TYPE, + le64_to_cpu(sinode->data_seq), 0, ind_locks); + return ret; +} + +/* + * A quick atomic sample of the last inode number that's been allocated. + */ +u64 scoutfs_last_ino(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->super; + u64 last; + + spin_lock(&sbi->next_ino_lock); + last = le64_to_cpu(super->next_ino); + spin_unlock(&sbi->next_ino_lock); + + return last; +} + +/* + * Return an allocated and unused inode number. Returns -ENOSPC if + * we're out of inode. + * + * Each parent directory has its own pool of free inode numbers. Items + * are sorted by their inode numbers as they're stored in segments. + * This will tend to group together files that are created in a + * directory at the same time in segments. Concurrent creation across + * different directories will be stored in their own regions. + * + * Inode numbers are never reclaimed. If the inode is evicted or we're + * unmounted the pending inode numbers will be lost. Asking for a + * relatively small number from the server each time will tend to + * minimize that loss while still being large enough for typical + * directory file counts. + */ +int scoutfs_alloc_ino(struct super_block *sb, bool is_dir, u64 *ino_ret) +{ + DECLARE_INODE_SB_INFO(sb, inf); + struct inode_allocator *ia; + u64 ino; + u64 nr; + int ret; + + ia = is_dir ? &inf->dir_ino_alloc : &inf->ino_alloc; + + spin_lock(&ia->lock); + + if (ia->nr == 0) { + spin_unlock(&ia->lock); + ret = scoutfs_client_alloc_inodes(sb, + SCOUTFS_LOCK_INODE_GROUP_NR * 10, + &ino, &nr); + if (ret < 0) + goto out; + spin_lock(&ia->lock); + if (ia->nr == 0) { + ia->ino = ino; + ia->nr = nr; + } + } + + *ino_ret = ia->ino++; + ia->nr--; + + spin_unlock(&ia->lock); + ret = 0; +out: + trace_scoutfs_alloc_ino(sb, ret, *ino_ret, ia->ino, ia->nr); + return ret; +} + +/* + * Allocate and initialize a new inode. The caller is responsible for + * creating links to it and updating it. @dir can be null. + */ +struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir, + umode_t mode, dev_t rdev, u64 ino, + struct scoutfs_lock *lock) +{ + struct scoutfs_inode_info *ci; + struct scoutfs_key key; + struct scoutfs_inode sinode; + struct inode *inode; + int ret; + + inode = new_inode(sb); + if (!inode) + return ERR_PTR(-ENOMEM); + + ci = SCOUTFS_I(inode); + ci->ino = ino; + ci->data_version = 0; + ci->online_blocks = 0; + ci->offline_blocks = 0; + ci->next_readdir_pos = SCOUTFS_DIRENT_FIRST_POS; + ci->next_xattr_id = 0; + ci->have_item = false; + atomic64_set(&ci->last_refreshed, lock->refresh_gen); + ci->flags = 0; + + scoutfs_inode_set_meta_seq(inode); + scoutfs_inode_set_data_seq(inode); + + inode->i_ino = ino; /* XXX overflow */ + inode_init_owner(inode, dir, mode); + inode_set_bytes(inode, 0); + inode->i_mtime = inode->i_atime = inode->i_ctime = CURRENT_TIME; + inode->i_rdev = rdev; + set_inode_ops(inode); + + store_inode(&sinode, inode); + init_inode_key(&key, scoutfs_ino(inode)); + + ret = scoutfs_item_create(sb, &key, &sinode, sizeof(sinode), lock); + if (ret) { + iput(inode); + return ERR_PTR(ret); + } + + return inode; +} + +static void init_orphan_key(struct scoutfs_key *key, u64 rid, u64 ino) +{ + *key = (struct scoutfs_key) { + .sk_zone = SCOUTFS_RID_ZONE, + .sko_rid = cpu_to_le64(rid), + .sk_type = SCOUTFS_ORPHAN_TYPE, + .sko_ino = cpu_to_le64(ino), + }; +} + +static int remove_orphan_item(struct super_block *sb, u64 ino) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_lock *lock = sbi->rid_lock; + struct scoutfs_key key; + int ret; + + init_orphan_key(&key, sbi->rid, ino); + + ret = scoutfs_item_delete(sb, &key, lock); + if (ret == -ENOENT) + ret = 0; + + return ret; +} + +/* + * Remove all the items associated with a given inode. This is only + * called once nlink has dropped to zero so we don't have to worry about + * dirents referencing the inode or link backrefs. Dropping nlink to 0 + * also created an orphan item. That orphan item will continue + * triggering attempts to finish previous partial deletion until all + * deletion is complete and the orphan item is removed. + */ +static int delete_inode_items(struct super_block *sb, u64 ino) +{ + struct scoutfs_lock *lock = NULL; + struct scoutfs_inode sinode; + struct scoutfs_key key; + LIST_HEAD(ind_locks); + bool release = false; + umode_t mode; + u64 ind_seq; + u64 size; + int ret; + + ret = scoutfs_lock_ino(sb, SCOUTFS_LOCK_WRITE, 0, ino, &lock); + if (ret) + return ret; + + init_inode_key(&key, ino); + + ret = scoutfs_item_lookup_exact(sb, &key, &sinode, sizeof(sinode), + lock); + if (ret < 0) { + if (ret == -ENOENT) + ret = 0; + goto out; + } + + /* XXX corruption, inode probably won't be freed without repair */ + if (le32_to_cpu(sinode.nlink)) { + scoutfs_warn(sb, "Dangling orphan item for inode %llu.", ino); + ret = -EIO; + goto out; + } + + mode = le32_to_cpu(sinode.mode); + size = le64_to_cpu(sinode.size); + trace_scoutfs_delete_inode(sb, ino, mode, size); + + /* remove data items in their own transactions */ + if (S_ISREG(mode)) { + ret = scoutfs_data_truncate_items(sb, NULL, ino, 0, ~0ULL, + false, lock); + if (ret) + goto out; + } + + ret = scoutfs_xattr_drop(sb, ino, lock); + if (ret) + goto out; + + /* then delete the small known number of remaining inode items */ +retry: + ret = scoutfs_inode_index_start(sb, &ind_seq) ?: + prepare_index_deletion(sb, &ind_locks, ino, mode, &sinode) ?: + scoutfs_inode_index_try_lock_hold(sb, &ind_locks, ind_seq, + SIC_DROP_INODE(mode, size)); + if (ret > 0) + goto retry; + if (ret) + goto out; + + release = true; + + ret = remove_index_items(sb, ino, &sinode, &ind_locks); + if (ret) + goto out; + + if (S_ISLNK(mode)) { + ret = scoutfs_symlink_drop(sb, ino, lock, size); + if (ret) + goto out; + } + + ret = scoutfs_item_delete(sb, &key, lock); + if (ret) + goto out; + + ret = remove_orphan_item(sb, ino); +out: + if (release) + scoutfs_release_trans(sb); + scoutfs_inode_index_unlock(sb, &ind_locks); + scoutfs_unlock(sb, lock, SCOUTFS_LOCK_WRITE); + return ret; +} + +/* + * iput_final has already written out the dirty pages to the inode + * before we get here. We're left with a clean inode that we have to + * tear down. If there are no more links to the inode then we also + * remove all its persistent structures. + */ +void scoutfs_evict_inode(struct inode *inode) +{ + trace_scoutfs_evict_inode(inode->i_sb, scoutfs_ino(inode), + inode->i_nlink, is_bad_inode(inode)); + + if (is_bad_inode(inode)) + goto clear; + + truncate_inode_pages_final(&inode->i_data); + + if (inode->i_nlink == 0) + delete_inode_items(inode->i_sb, scoutfs_ino(inode)); +clear: + clear_inode(inode); +} + +int scoutfs_drop_inode(struct inode *inode) +{ + int ret = generic_drop_inode(inode); + + trace_scoutfs_drop_inode(inode->i_sb, scoutfs_ino(inode), + inode->i_nlink, inode_unhashed(inode)); + return ret; +} + +/* + * Find orphan items and process each one. + * + * Runtime of this will be bounded by the number of orphans, which could + * theoretically be very large. If that becomes a problem we might want to push + * this work off to a thread. + * + * This only scans orphans for this node. This will need to be covered by + * the rest of node zone cleanup. + */ +int scoutfs_scan_orphans(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_lock *lock = sbi->rid_lock; + struct scoutfs_key key; + struct scoutfs_key last; + int err = 0; + int ret; + + trace_scoutfs_scan_orphans(sb); + + init_orphan_key(&key, sbi->rid, 0); + init_orphan_key(&last, sbi->rid, ~0ULL); + + while (1) { + ret = scoutfs_item_next(sb, &key, &last, NULL, 0, lock); + if (ret == -ENOENT) /* No more orphan items */ + break; + if (ret < 0) + goto out; + + ret = delete_inode_items(sb, le64_to_cpu(key.sko_ino)); + if (ret && ret != -ENOENT && !err) + err = ret; + + if (le64_to_cpu(key.sko_ino) == U64_MAX) { + ret = -ENOENT; + break; + } + le64_add_cpu(&key.sko_ino, 1); + } + + ret = 0; +out: + return err ? err : ret; +} + +int scoutfs_orphan_inode(struct inode *inode) +{ + struct super_block *sb = inode->i_sb; + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_lock *lock = sbi->rid_lock; + struct scoutfs_key key; + int ret; + + trace_scoutfs_orphan_inode(sb, inode); + + init_orphan_key(&key, sbi->rid, scoutfs_ino(inode)); + + ret = scoutfs_item_create(sb, &key, NULL, 0, lock); + + return ret; +} + +/* + * Track an inode that could have dirty pages. Used to kick off writeback + * on all dirty pages during transaction commit without tying ourselves in + * knots trying to call through the high level vfs sync methods. + */ +void scoutfs_inode_queue_writeback(struct inode *inode) +{ + DECLARE_INODE_SB_INFO(inode->i_sb, inf); + struct scoutfs_inode_info *si = SCOUTFS_I(inode); + + spin_lock(&inf->writeback_lock); + if (RB_EMPTY_NODE(&si->writeback_node)) + insert_writeback_inode(inf, si); + spin_unlock(&inf->writeback_lock); +} + +/* + * Walk our dirty inodes in ino order and either start dirty page + * writeback or wait for writeback to complete. + * + * This is called by transaction commiting so other writers are + * excluded. We're still very careful to iterate over the tree while it + * and the inodes could be changing. + * + * Because writes are excluded we know that there's no remaining dirty + * pages once waiting returns successfully. + * + * XXX not sure what to do about retrying io errors. + */ +int scoutfs_inode_walk_writeback(struct super_block *sb, bool write) +{ + DECLARE_INODE_SB_INFO(sb, inf); + struct scoutfs_inode_info *si; + struct rb_node *node; + struct inode *inode; + struct inode *defer_iput = NULL; + int ret; + + spin_lock(&inf->writeback_lock); + + node = rb_first(&inf->writeback_inodes); + while (node) { + si = container_of(node, struct scoutfs_inode_info, + writeback_node); + node = rb_next(node); + inode = igrab(&si->inode); + if (!inode) + continue; + + spin_unlock(&inf->writeback_lock); + + if (defer_iput) { + iput(defer_iput); + defer_iput = NULL; + } + + if (write) + ret = filemap_fdatawrite(inode->i_mapping); + else + ret = filemap_fdatawait(inode->i_mapping); + trace_scoutfs_inode_walk_writeback(sb, scoutfs_ino(inode), + write, ret); + if (ret) { + iput(inode); + goto out; + } + + spin_lock(&inf->writeback_lock); + + if (WARN_ON_ONCE(RB_EMPTY_NODE(&si->writeback_node))) + node = rb_first(&inf->writeback_inodes); + else + node = rb_next(&si->writeback_node); + + if (!write) + remove_writeback_inode(inf, si); + + /* avoid iput->destroy lock deadlock */ + defer_iput = inode; + } + + spin_unlock(&inf->writeback_lock); +out: + if (defer_iput) + iput(defer_iput); + return ret; +} + +int scoutfs_inode_setup(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct inode_sb_info *inf; + + inf = kzalloc(sizeof(struct inode_sb_info), GFP_KERNEL); + if (!inf) + return -ENOMEM; + + spin_lock_init(&inf->writeback_lock); + inf->writeback_inodes = RB_ROOT; + spin_lock_init(&inf->dir_ino_alloc.lock); + spin_lock_init(&inf->ino_alloc.lock); + + sbi->inode_sb_info = inf; + + return 0; +} + +void scoutfs_inode_destroy(struct super_block *sb) +{ + struct inode_sb_info *inf = SCOUTFS_SB(sb)->inode_sb_info; + + kfree(inf); +} + +void scoutfs_inode_exit(void) +{ + if (scoutfs_inode_cachep) { + rcu_barrier(); + kmem_cache_destroy(scoutfs_inode_cachep); + scoutfs_inode_cachep = NULL; + } +} + +int scoutfs_inode_init(void) +{ + scoutfs_inode_cachep = kmem_cache_create("scoutfs_inode_info", + sizeof(struct scoutfs_inode_info), 0, + SLAB_RECLAIM_ACCOUNT, + scoutfs_inode_ctor); + if (!scoutfs_inode_cachep) + return -ENOMEM; + + return 0; +} diff --git a/kmod/src/inode.h b/kmod/src/inode.h new file mode 100644 index 00000000..9034aef4 --- /dev/null +++ b/kmod/src/inode.h @@ -0,0 +1,124 @@ +#ifndef _SCOUTFS_INODE_H_ +#define _SCOUTFS_INODE_H_ + +#include "key.h" +#include "lock.h" +#include "per_task.h" +#include "count.h" +#include "format.h" +#include "data.h" + +struct scoutfs_lock; + +struct scoutfs_inode_info { + /* read or initialized for each inode instance */ + u64 ino; + u64 next_readdir_pos; + u64 next_xattr_id; + u64 meta_seq; + u64 data_seq; + u64 data_version; + u64 online_blocks; + u64 offline_blocks; + u32 flags; + + /* + * The in-memory item info caches the current index item values + * so that we can decide to update them with comparisons instead + * of by maintaining state that tracks the inode differing from + * the item. The "item_" prefix is a bit clumsy :/. + */ + struct mutex item_mutex; + bool have_item; + u64 item_majors[SCOUTFS_INODE_INDEX_NR]; + u32 item_minors[SCOUTFS_INODE_INDEX_NR]; + + /* updated at on each new lock acquisition */ + atomic64_t last_refreshed; + + /* initialized once for slab object */ + seqcount_t seqcount; + bool staging; /* holder of i_mutex is staging */ + struct scoutfs_per_task pt_data_lock; + struct scoutfs_data_waitq data_waitq; + struct rw_semaphore xattr_rwsem; + struct rb_node writeback_node; + + struct inode inode; +}; + +static inline struct scoutfs_inode_info *SCOUTFS_I(struct inode *inode) +{ + return container_of(inode, struct scoutfs_inode_info, inode); +} + +static inline u64 scoutfs_ino(struct inode *inode) +{ + return SCOUTFS_I(inode)->ino; +} + +struct inode *scoutfs_alloc_inode(struct super_block *sb); +void scoutfs_destroy_inode(struct inode *inode); +int scoutfs_drop_inode(struct inode *inode); +void scoutfs_evict_inode(struct inode *inode); +int scoutfs_orphan_inode(struct inode *inode); + +struct inode *scoutfs_iget(struct super_block *sb, u64 ino); +struct inode *scoutfs_ilookup(struct super_block *sb, u64 ino); + +void scoutfs_inode_init_index_key(struct scoutfs_key *key, u8 type, u64 major, + u32 minor, u64 ino); +int scoutfs_inode_index_start(struct super_block *sb, u64 *seq); +int scoutfs_inode_index_prepare(struct super_block *sb, struct list_head *list, + struct inode *inode, bool set_data_seq); +int scoutfs_inode_index_prepare_ino(struct super_block *sb, + struct list_head *list, u64 ino, + umode_t mode); +int scoutfs_inode_index_try_lock_hold(struct super_block *sb, + struct list_head *list, u64 seq, + const struct scoutfs_item_count cnt); +int scoutfs_inode_index_lock_hold(struct inode *inode, struct list_head *list, + bool set_data_seq, + const struct scoutfs_item_count cnt); +void scoutfs_inode_index_unlock(struct super_block *sb, struct list_head *list); + +int scoutfs_dirty_inode_item(struct inode *inode, struct scoutfs_lock *lock); +void scoutfs_update_inode_item(struct inode *inode, struct scoutfs_lock *lock, + struct list_head *ind_locks); + +int scoutfs_alloc_ino(struct super_block *sb, bool is_dir, u64 *ino_ret); +struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir, + umode_t mode, dev_t rdev, u64 ino, + struct scoutfs_lock *lock); + +void scoutfs_inode_set_meta_seq(struct inode *inode); +void scoutfs_inode_set_data_seq(struct inode *inode); +void scoutfs_inode_inc_data_version(struct inode *inode); +void scoutfs_inode_set_data_version(struct inode *inode, u64 data_version); +void scoutfs_inode_add_onoff(struct inode *inode, s64 on, s64 off); +u64 scoutfs_inode_meta_seq(struct inode *inode); +u64 scoutfs_inode_data_seq(struct inode *inode); +u64 scoutfs_inode_data_version(struct inode *inode); +void scoutfs_inode_get_onoff(struct inode *inode, s64 *on, s64 *off); +int scoutfs_complete_truncate(struct inode *inode, struct scoutfs_lock *lock); + +int scoutfs_inode_refresh(struct inode *inode, struct scoutfs_lock *lock, + int flags); +int scoutfs_getattr(struct vfsmount *mnt, struct dentry *dentry, + struct kstat *stat); +int scoutfs_setattr(struct dentry *dentry, struct iattr *attr); + +int scoutfs_scan_orphans(struct super_block *sb); + +void scoutfs_inode_queue_writeback(struct inode *inode); +int scoutfs_inode_walk_writeback(struct super_block *sb, bool write); + +u64 scoutfs_last_ino(struct super_block *sb); + +void scoutfs_inode_exit(void); +int scoutfs_inode_init(void); + +int scoutfs_inode_setup(struct super_block *sb); +void scoutfs_inode_destroy(struct super_block *sb); + +#endif diff --git a/kmod/src/ioctl.c b/kmod/src/ioctl.c new file mode 100644 index 00000000..cfb06462 --- /dev/null +++ b/kmod/src/ioctl.c @@ -0,0 +1,966 @@ +/* + * Copyright (C) 2016 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "format.h" +#include "key.h" +#include "dir.h" +#include "ioctl.h" +#include "super.h" +#include "inode.h" +#include "item.h" +#include "forest.h" +#include "data.h" +#include "client.h" +#include "lock.h" +#include "trans.h" +#include "xattr.h" +#include "hash.h" +#include "srch.h" +#include "alloc.h" +#include "scoutfs_trace.h" + +/* + * We make inode index items coherent by locking fixed size regions of + * the key space. But the inode index item key space is vast and can + * have huge sparse regions. To avoid trying every possible lock in the + * sparse regions we use the manifest to find the next stable key in the + * key space after we find no items in a given lock region. This is + * relatively cheap because reading is going to check the segments + * anyway. + * + * This is copying to userspace while holding a read lock. This is safe + * because faulting can send a request for a write lock while the read + * lock is being used. The cluster locks don't block tasks in a node, + * they match and the tasks fall back to local locking. In this case + * the spin locks around the item cache. + */ +static long scoutfs_ioc_walk_inodes(struct file *file, unsigned long arg) +{ + struct super_block *sb = file_inode(file)->i_sb; + struct scoutfs_ioctl_walk_inodes __user *uwalk = (void __user *)arg; + struct scoutfs_ioctl_walk_inodes walk; + struct scoutfs_ioctl_walk_inodes_entry ent; + struct scoutfs_key next_key; + struct scoutfs_key last_key; + struct scoutfs_key key; + struct scoutfs_lock *lock; + u64 last_seq; + int ret = 0; + u32 nr = 0; + u8 type; + + if (copy_from_user(&walk, uwalk, sizeof(walk))) + return -EFAULT; + + trace_scoutfs_ioc_walk_inodes(sb, &walk); + + if (walk.index == SCOUTFS_IOC_WALK_INODES_META_SEQ) + type = SCOUTFS_INODE_INDEX_META_SEQ_TYPE; + else if (walk.index == SCOUTFS_IOC_WALK_INODES_DATA_SEQ) + type = SCOUTFS_INODE_INDEX_DATA_SEQ_TYPE; + else + return -EINVAL; + + /* clamp results to the inodes in the farthest stable seq */ + if (type == SCOUTFS_INODE_INDEX_META_SEQ_TYPE || + type == SCOUTFS_INODE_INDEX_DATA_SEQ_TYPE) { + + ret = scoutfs_client_get_last_seq(sb, &last_seq); + if (ret) + return ret; + + if (last_seq < walk.last.major) { + walk.last.major = last_seq; + walk.last.minor = ~0; + walk.last.ino = ~0ULL; + } + } + + scoutfs_inode_init_index_key(&key, type, walk.first.major, + walk.first.minor, walk.first.ino); + scoutfs_inode_init_index_key(&last_key, type, walk.last.major, + walk.last.minor, walk.last.ino); + + /* cap nr to the max the ioctl can return to a compat task */ + walk.nr_entries = min_t(u64, walk.nr_entries, INT_MAX); + + ret = scoutfs_lock_inode_index(sb, SCOUTFS_LOCK_READ, type, + walk.first.major, walk.first.ino, + &lock); + if (ret < 0) + goto out; + + for (nr = 0; nr < walk.nr_entries; ) { + + ret = scoutfs_item_next(sb, &key, &last_key, NULL, 0, lock); + if (ret < 0 && ret != -ENOENT) + break; + + if (ret == -ENOENT) { + + /* done if lock covers last iteration key */ + if (scoutfs_key_compare(&last_key, &lock->end) <= 0) { + ret = 0; + break; + } + + /* continue iterating after locked empty region */ + key = lock->end; + scoutfs_key_inc(&key); + + scoutfs_unlock(sb, lock, SCOUTFS_LOCK_READ); + + ret = scoutfs_forest_next_hint(sb, &key, &next_key); + if (ret < 0 && ret != -ENOENT) + goto out; + + if (ret == -ENOENT || + scoutfs_key_compare(&next_key, &last_key) > 0) { + ret = 0; + goto out; + } + + key = next_key; + + ret = scoutfs_lock_inode_index(sb, SCOUTFS_LOCK_READ, + key.sk_type, + le64_to_cpu(key.skii_major), + le64_to_cpu(key.skii_ino), + &lock); + if (ret < 0) + goto out; + + continue; + } + + ent.major = le64_to_cpu(key.skii_major); + ent.minor = 0; + ent.ino = le64_to_cpu(key.skii_ino); + + if (copy_to_user((void __user *)walk.entries_ptr, &ent, + sizeof(ent))) { + ret = -EFAULT; + break; + } + + nr++; + walk.entries_ptr += sizeof(ent); + + scoutfs_key_inc(&key); + } + + scoutfs_unlock(sb, lock, SCOUTFS_LOCK_READ); + +out: + if (nr > 0) + ret = nr; + + return ret; +} + +/* + * See the comment above the definition of struct scoutfs_ioctl_ino_path + * for ioctl semantics. + */ +static long scoutfs_ioc_ino_path(struct file *file, unsigned long arg) +{ + struct super_block *sb = file_inode(file)->i_sb; + struct scoutfs_ioctl_ino_path_result __user *ures; + struct scoutfs_link_backref_entry *last_ent; + struct scoutfs_link_backref_entry *ent; + struct scoutfs_ioctl_ino_path args; + LIST_HEAD(list); + u16 copied; + char term; + int ret; + + if (!capable(CAP_DAC_READ_SEARCH)) + return -EPERM; + + if (copy_from_user(&args, (void __user *)arg, sizeof(args))) + return -EFAULT; + + ures = (void __user *)(unsigned long)args.result_ptr; + + ret = scoutfs_dir_get_backref_path(sb, args.ino, args.dir_ino, + args.dir_pos, &list); + if (ret < 0) + goto out; + + last_ent = list_last_entry(&list, struct scoutfs_link_backref_entry, + head); + copied = 0; + list_for_each_entry(ent, &list, head) { + + if (offsetof(struct scoutfs_ioctl_ino_path_result, + path[copied + ent->name_len + 1]) + > args.result_bytes) { + ret = -ENAMETOOLONG; + goto out; + } + + if (copy_to_user(&ures->path[copied], + ent->dent.name, ent->name_len)) { + ret = -EFAULT; + goto out; + } + + copied += ent->name_len; + + if (ent == last_ent) + term = '\0'; + else + term = '/'; + + if (put_user(term, &ures->path[copied])) { + ret = -EFAULT; + break; + } + + copied++; + } + + /* fill the result header now that we know the copied path length */ + if (put_user(last_ent->dir_ino, &ures->dir_ino) || + put_user(last_ent->dir_pos, &ures->dir_pos) || + put_user(copied, &ures->path_bytes)) { + ret = -EFAULT; + } else { + ret = 0; + } + +out: + scoutfs_dir_free_backref_path(sb, &list); + return ret; +} + +/* + * The caller has a version of the data available in the given byte + * range in an external archive. As long as the data version still + * matches we free the blocks fully contained in the range and mark them + * offline. Attempts to use the blocks in the future will trigger + * recall from the archive. + * + * If the file's online blocks drop to 0 then we also truncate any + * blocks beyond i_size. This honors the intent of fully releasing a file + * without the user needing to know to release past i_size or truncate. + * + * XXX permissions? + * XXX a lot of this could be generic file write prep + */ +static long scoutfs_ioc_release(struct file *file, unsigned long arg) +{ + struct inode *inode = file_inode(file); + struct super_block *sb = inode->i_sb; + struct scoutfs_ioctl_release args; + struct scoutfs_lock *lock = NULL; + loff_t start; + loff_t end_inc; + u64 online; + u64 offline; + u64 isize; + int ret; + + if (copy_from_user(&args, (void __user *)arg, sizeof(args))) + return -EFAULT; + + trace_scoutfs_ioc_release(sb, scoutfs_ino(inode), &args); + + if (args.count == 0) + return 0; + if ((args.block + args.count) < args.block) + return -EINVAL; + + + ret = mnt_want_write_file(file); + if (ret) + return ret; + + mutex_lock(&inode->i_mutex); + + ret = scoutfs_lock_inode(sb, SCOUTFS_LOCK_WRITE, + SCOUTFS_LKF_REFRESH_INODE, inode, &lock); + if (ret) + goto out; + + if (!S_ISREG(inode->i_mode)) { + ret = -EINVAL; + goto out; + } + + if (!(file->f_mode & FMODE_WRITE)) { + ret = -EINVAL; + goto out; + } + + if (scoutfs_inode_data_version(inode) != args.data_version) { + ret = -ESTALE; + goto out; + } + + inode_dio_wait(inode); + + /* drop all clean and dirty cached blocks in the range */ + start = args.block << SCOUTFS_BLOCK_SM_SHIFT; + end_inc = ((args.block + args.count) << SCOUTFS_BLOCK_SM_SHIFT) - 1; + truncate_inode_pages_range(&inode->i_data, start, end_inc); + + ret = scoutfs_data_truncate_items(sb, inode, scoutfs_ino(inode), + args.block, + args.block + args.count - 1, true, + lock); + if (ret == 0) { + scoutfs_inode_get_onoff(inode, &online, &offline); + isize = i_size_read(inode); + if (online == 0 && isize) { + start = (isize + SCOUTFS_BLOCK_SM_SIZE - 1) + >> SCOUTFS_BLOCK_SM_SHIFT; + ret = scoutfs_data_truncate_items(sb, inode, + scoutfs_ino(inode), + start, U64_MAX, + false, lock); + } + } + +out: + scoutfs_unlock(sb, lock, SCOUTFS_LOCK_WRITE); + mutex_unlock(&inode->i_mutex); + mnt_drop_write_file(file); + + trace_scoutfs_ioc_release_ret(sb, scoutfs_ino(inode), ret); + return ret; +} + +static long scoutfs_ioc_data_wait_err(struct file *file, unsigned long arg) +{ + struct super_block *sb = file_inode(file)->i_sb; + struct scoutfs_ioctl_data_wait_err args; + struct scoutfs_lock *lock = NULL; + struct inode *inode = NULL; + u64 sblock; + u64 eblock; + long ret; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + if (copy_from_user(&args, (void __user *)arg, sizeof(args))) + return -EFAULT; + if (args.count == 0) + return 0; + if ((args.op & SCOUTFS_IOC_DWO_UNKNOWN) || !IS_ERR_VALUE(args.err)) + return -EINVAL; + if ((args.op & SCOUTFS_IOC_DWO_UNKNOWN) || !IS_ERR_VALUE(args.err)) + return -EINVAL; + + trace_scoutfs_ioc_data_wait_err(sb, &args); + + sblock = args.offset >> SCOUTFS_BLOCK_SM_SHIFT; + eblock = (args.offset + args.count - 1) >> SCOUTFS_BLOCK_SM_SHIFT; + + if (sblock > eblock) + return -EINVAL; + + inode = scoutfs_ilookup(sb, args.ino); + if (!inode) { + ret = -ESTALE; + goto out; + } + + mutex_lock(&inode->i_mutex); + + ret = scoutfs_lock_inode(sb, SCOUTFS_LOCK_READ, + SCOUTFS_LKF_REFRESH_INODE, inode, &lock); + if (ret) + goto unlock; + + if (!S_ISREG(inode->i_mode)) { + ret = -EINVAL; + } else if (scoutfs_inode_data_version(inode) != args.data_version) { + ret = -ESTALE; + } else { + ret = scoutfs_data_wait_err(inode, sblock, eblock, args.op, + args.err); + } + + scoutfs_unlock(sb, lock, SCOUTFS_LOCK_READ); +unlock: + mutex_unlock(&inode->i_mutex); + iput(inode); +out: + return ret; +} + +/* + * Write the archived contents of the file back if the data_version + * still matches. + * + * This is a data plane operation only. We don't want the write to + * change any fields in the inode. It only changes the file contents. + * + * Keep in mind that the staging writes can easily span transactions and + * can crash partway through. If we called the normal write path and + * restored the inode afterwards the modified inode could be commited + * partway through by a transaction and then left that way by a crash + * before the write finishes and we restore the fields. It also + * wouldn't be great if the temporarily updated inode was visible to + * paths that don't serialize with write. + * + * We're implementing the buffered write path down to the start of + * generic_file_buffered_writes() without all the stuff that would + * change the inode: file_remove_suid(), file_update_time(). The + * easiest way to do that is to call generic_file_buffered_write(). + * We're careful to only allow staging writes inside i_size. + * + * We set a bool on the inode which tells our code to update the + * offline extents and to not update the data_version counter. + * + * This doesn't support any fancy write modes or side-effects: aio, + * direct, append, sync, breaking suid, sending rlimit signals. + */ +static long scoutfs_ioc_stage(struct file *file, unsigned long arg) +{ + struct inode *inode = file_inode(file); + struct super_block *sb = inode->i_sb; + struct address_space *mapping = inode->i_mapping; + struct scoutfs_inode_info *si = SCOUTFS_I(inode); + SCOUTFS_DECLARE_PER_TASK_ENTRY(pt_ent); + struct scoutfs_ioctl_stage args; + struct scoutfs_lock *lock = NULL; + struct kiocb kiocb; + struct iovec iov; + size_t written; + loff_t end_size; + loff_t isize; + loff_t pos; + int ret; + + if (copy_from_user(&args, (void __user *)arg, sizeof(args))) + return -EFAULT; + + trace_scoutfs_ioc_stage(sb, scoutfs_ino(inode), &args); + + end_size = args.offset + args.count; + + /* verify arg constraints that aren't dependent on file */ + if (args.count < 0 || (end_size < args.offset) || + args.offset & SCOUTFS_BLOCK_SM_MASK) + return -EINVAL; + + if (args.count == 0) + return 0; + + /* the iocb is really only used for the file pointer :P */ + init_sync_kiocb(&kiocb, file); + kiocb.ki_pos = args.offset; + kiocb.ki_left = args.count; + kiocb.ki_nbytes = args.count; + iov.iov_base = (void __user *)(unsigned long)args.buf_ptr; + iov.iov_len = args.count; + + ret = mnt_want_write_file(file); + if (ret) + return ret; + + mutex_lock(&inode->i_mutex); + + ret = scoutfs_lock_inode(sb, SCOUTFS_LOCK_WRITE, + SCOUTFS_LKF_REFRESH_INODE, inode, &lock); + if (ret) + goto out; + + scoutfs_per_task_add(&si->pt_data_lock, &pt_ent, lock); + + isize = i_size_read(inode); + + if (!S_ISREG(inode->i_mode) || + !(file->f_mode & FMODE_WRITE) || + (file->f_flags & (O_APPEND | O_DIRECT | O_DSYNC)) || + IS_SYNC(file->f_mapping->host) || + (end_size > isize) || + ((end_size & SCOUTFS_BLOCK_SM_MASK) && (end_size != isize))) { + ret = -EINVAL; + goto out; + } + + if (scoutfs_inode_data_version(inode) != args.data_version) { + ret = -ESTALE; + goto out; + } + + si->staging = true; + current->backing_dev_info = mapping->backing_dev_info; + + pos = args.offset; + written = 0; + do { + ret = generic_file_buffered_write(&kiocb, &iov, 1, pos, &pos, + args.count, written); + BUG_ON(ret == -EIOCBQUEUED); + if (ret > 0) + written += ret; + } while (ret > 0 && written < args.count); + + si->staging = false; + current->backing_dev_info = NULL; +out: + scoutfs_per_task_del(&si->pt_data_lock, &pt_ent); + scoutfs_unlock(sb, lock, SCOUTFS_LOCK_WRITE); + mutex_unlock(&inode->i_mutex); + mnt_drop_write_file(file); + + trace_scoutfs_ioc_stage_ret(sb, scoutfs_ino(inode), ret); + return ret; +} + +static long scoutfs_ioc_stat_more(struct file *file, unsigned long arg) +{ + struct inode *inode = file_inode(file); + struct scoutfs_ioctl_stat_more stm; + + if (get_user(stm.valid_bytes, (__u64 __user *)arg)) + return -EFAULT; + + stm.valid_bytes = min_t(u64, stm.valid_bytes, + sizeof(struct scoutfs_ioctl_stat_more)); + stm.meta_seq = scoutfs_inode_meta_seq(inode); + stm.data_seq = scoutfs_inode_data_seq(inode); + stm.data_version = scoutfs_inode_data_version(inode); + scoutfs_inode_get_onoff(inode, &stm.online_blocks, &stm.offline_blocks); + + if (copy_to_user((void __user *)arg, &stm, stm.valid_bytes)) + return -EFAULT; + + return 0; +} + +static bool inc_wrapped(u64 *ino, u64 *iblock) +{ + return (++(*iblock) == 0) && (++(*ino) == 0); +} + +static long scoutfs_ioc_data_waiting(struct file *file, unsigned long arg) +{ + struct super_block *sb = file_inode(file)->i_sb; + struct scoutfs_ioctl_data_waiting idw; + struct scoutfs_ioctl_data_waiting_entry __user *udwe; + struct scoutfs_ioctl_data_waiting_entry dwe[16]; + unsigned int nr; + int total; + int ret; + + if (copy_from_user(&idw, (void __user *)arg, sizeof(idw))) + return -EFAULT; + + if (idw.flags & SCOUTFS_IOC_DATA_WAITING_FLAGS_UNKNOWN) + return -EINVAL; + + udwe = (void __user *)(long)idw.ents_ptr; + total = 0; + ret = 0; + while (idw.ents_nr && !inc_wrapped(&idw.after_ino, &idw.after_iblock)) { + nr = min_t(size_t, idw.ents_nr, ARRAY_SIZE(dwe)); + + ret = scoutfs_data_waiting(sb, idw.after_ino, idw.after_iblock, + dwe, nr); + BUG_ON(ret > nr); /* stack overflow \o/ */ + if (ret <= 0) + break; + + if (copy_to_user(udwe, dwe, ret * sizeof(dwe[0]))) { + ret = -EFAULT; + break; + } + + idw.after_ino = dwe[ret - 1].ino; + idw.after_iblock = dwe[ret - 1].iblock; + + udwe += ret; + idw.ents_nr -= ret; + total += ret; + ret = 0; + } + + return ret ?: total; +} + +/* + * This is used when restoring files, it lets the caller set all the + * inode attributes which are otherwise unreachable. Changing the file + * size can only be done for regular files with a data_version of 0. + */ +static long scoutfs_ioc_setattr_more(struct file *file, unsigned long arg) +{ + struct inode *inode = file->f_inode; + struct super_block *sb = inode->i_sb; + struct scoutfs_ioctl_setattr_more __user *usm = (void __user *)arg; + struct scoutfs_ioctl_setattr_more sm; + struct scoutfs_lock *lock = NULL; + LIST_HEAD(ind_locks); + bool set_data_seq; + int ret; + + if (!capable(CAP_SYS_ADMIN)) { + ret = -EPERM; + goto out; + } + + if (!(file->f_mode & FMODE_WRITE)) { + ret = -EBADF; + goto out; + } + + if (copy_from_user(&sm, usm, sizeof(sm))) { + ret = -EFAULT; + goto out; + } + + if ((sm.i_size > 0 && sm.data_version == 0) || + ((sm.flags & SCOUTFS_IOC_SETATTR_MORE_OFFLINE) && !sm.i_size) || + (sm.flags & SCOUTFS_IOC_SETATTR_MORE_UNKNOWN)) { + ret = -EINVAL; + goto out; + } + + ret = mnt_want_write_file(file); + if (ret) + goto out; + + mutex_lock(&inode->i_mutex); + + ret = scoutfs_lock_inode(sb, SCOUTFS_LOCK_WRITE, + SCOUTFS_LKF_REFRESH_INODE, inode, &lock); + if (ret) + goto unlock; + + /* can only change size/dv on untouched regular files */ + if ((sm.i_size != 0 || sm.data_version != 0) && + ((!S_ISREG(inode->i_mode) || + scoutfs_inode_data_version(inode) != 0))) { + ret = -EINVAL; + goto unlock; + } + + /* create offline extents in potentially many transactions */ + if (sm.flags & SCOUTFS_IOC_SETATTR_MORE_OFFLINE) { + ret = scoutfs_data_init_offline_extent(inode, sm.i_size, lock); + if (ret) + goto unlock; + } + + /* setting only so we don't see 0 data seq with nonzero data_version */ + set_data_seq = sm.data_version != 0 ? true : false; + ret = scoutfs_inode_index_lock_hold(inode, &ind_locks, set_data_seq, + SIC_SETATTR_MORE()); + if (ret) + goto unlock; + + if (sm.data_version) + scoutfs_inode_set_data_version(inode, sm.data_version); + if (sm.i_size) + i_size_write(inode, sm.i_size); + inode->i_ctime.tv_sec = sm.ctime_sec; + inode->i_ctime.tv_nsec = sm.ctime_nsec; + + scoutfs_update_inode_item(inode, lock, &ind_locks); + ret = 0; + + scoutfs_release_trans(sb); +unlock: + scoutfs_inode_index_unlock(sb, &ind_locks); + scoutfs_unlock(sb, lock, SCOUTFS_LOCK_WRITE); + mutex_unlock(&inode->i_mutex); + mnt_drop_write_file(file); +out: + + return ret; +} + +/* + * This lists .hide. attributes on the inode. It doesn't include normal + * xattrs that are visible to listxattr because we don't perform as + * rigorous security access checks as normal vfs listxattr does. + */ +static long scoutfs_ioc_listxattr_hidden(struct file *file, unsigned long arg) +{ + struct inode *inode = file->f_inode; + struct scoutfs_ioctl_listxattr_hidden __user *ulxr = (void __user *)arg; + struct scoutfs_ioctl_listxattr_hidden lxh; + struct page *page = NULL; + unsigned int bytes; + int total = 0; + int ret; + + ret = inode_permission(inode, MAY_READ); + if (ret < 0) + goto out; + + if (copy_from_user(&lxh, ulxr, sizeof(lxh))) { + ret = -EFAULT; + goto out; + } + + page = alloc_page(GFP_KERNEL); + if (!page) { + ret = -ENOMEM; + goto out; + } + + while (lxh.buf_bytes) { + bytes = min_t(int, lxh.buf_bytes, PAGE_SIZE); + ret = scoutfs_list_xattrs(inode, page_address(page), bytes, + &lxh.hash_pos, &lxh.id_pos, + false, true); + if (ret <= 0) + break; + + if (copy_to_user((void __user *)lxh.buf_ptr, + page_address(page), ret)) { + ret = -EFAULT; + break; + } + + lxh.buf_ptr += ret; + lxh.buf_bytes -= ret; + total += ret; + ret = 0; + } + +out: + if (page) + __free_page(page); + + if (ret == 0 && (__put_user(lxh.hash_pos, &ulxr->hash_pos) || + __put_user(lxh.id_pos, &ulxr->id_pos))) + ret = -EFAULT; + + return ret ?: total; +} + +/* + * Return the inode numbers of inodes which might contain the given + * named xattr. This will only find scoutfs xattrs with the index tag + * but we don't check that the callers xattr name contains the tag and + * search for it regardless. + */ +static long scoutfs_ioc_search_xattrs(struct file *file, unsigned long arg) +{ + struct super_block *sb = file_inode(file)->i_sb; + struct scoutfs_ioctl_search_xattrs __user *usx = (void __user *)arg; + struct scoutfs_ioctl_search_xattrs sx; + struct scoutfs_xattr_prefix_tags tgs; + struct scoutfs_srch_rb_root sroot; + struct scoutfs_srch_rb_node *snode; + u64 __user *uinos; + struct rb_node *node; + char *name = NULL; + bool done = false; + u64 prev_ino = 0; + u64 total = 0; + int ret; + + if (!(file->f_mode & FMODE_READ)) { + ret = -EBADF; + goto out; + } + + if (!capable(CAP_SYS_ADMIN)) { + ret = -EPERM; + goto out; + } + + if (copy_from_user(&sx, usx, sizeof(sx))) { + ret = -EFAULT; + goto out; + } + uinos = (u64 __user *)sx.inodes_ptr; + + if (sx.name_bytes > SCOUTFS_XATTR_MAX_NAME_LEN) { + ret = -EINVAL; + goto out; + } + + if (sx.nr_inodes == 0 || sx.last_ino < sx.next_ino) { + ret = 0; + goto out; + } + + name = kmalloc(sx.name_bytes, GFP_KERNEL); + if (!name) { + ret = -ENOMEM; + goto out; + } + + if (copy_from_user(name, (void __user *)sx.name_ptr, sx.name_bytes)) { + ret = -EFAULT; + goto out; + } + + if (scoutfs_xattr_parse_tags(name, sx.name_bytes, &tgs) < 0 || + !tgs.srch) { + ret = -EINVAL; + goto out; + } + + ret = scoutfs_srch_search_xattrs(sb, &sroot, + scoutfs_hash64(name, sx.name_bytes), + sx.next_ino, sx.last_ino, &done); + if (ret < 0) + goto out; + + prev_ino = 0; + scoutfs_srch_foreach_rb_node(snode, node, &sroot) { + if (prev_ino == snode->ino) + continue; + + if (put_user(snode->ino, uinos + total)) { + ret = -EFAULT; + break; + } + prev_ino = snode->ino; + + if (++total == sx.nr_inodes) + break; + } + + sx.output_flags = 0; + if (done && total == sroot.nr) + sx.output_flags |= SCOUTFS_SEARCH_XATTRS_OFLAG_END; + + if (put_user(sx.output_flags, &usx->output_flags)) + ret = -EFAULT; + else + ret = 0; + + scoutfs_srch_destroy_rb_root(&sroot); + +out: + kfree(name); + return ret ?: total; +} + +static long scoutfs_ioc_statfs_more(struct file *file, unsigned long arg) +{ + struct super_block *sb = file_inode(file)->i_sb; + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->super; + struct scoutfs_ioctl_statfs_more sfm; + int ret; + + if (get_user(sfm.valid_bytes, (__u64 __user *)arg)) + return -EFAULT; + + sfm.valid_bytes = min_t(u64, sfm.valid_bytes, + sizeof(struct scoutfs_ioctl_statfs_more)); + sfm.fsid = le64_to_cpu(super->hdr.fsid); + sfm.rid = sbi->rid; + sfm.total_meta_blocks = le64_to_cpu(super->total_meta_blocks); + sfm.total_data_blocks = le64_to_cpu(super->total_data_blocks); + + ret = scoutfs_client_get_last_seq(sb, &sfm.committed_seq); + if (ret) + return ret; + + if (copy_to_user((void __user *)arg, &sfm, sfm.valid_bytes)) + return -EFAULT; + + return 0; +} + +struct copy_alloc_detail_args { + struct scoutfs_ioctl_alloc_detail_entry __user *uade; + u64 nr; + u64 copied; +}; + +static int copy_alloc_detail_to_user(struct super_block *sb, void *arg, + int owner, u64 id, bool meta, bool avail, + u64 blocks) +{ + struct copy_alloc_detail_args *args = arg; + struct scoutfs_ioctl_alloc_detail_entry ade; + + if (args->copied == args->nr) + return -EOVERFLOW; + + ade.blocks = blocks; + ade.id = id; + ade.meta = !!meta; + ade.avail = !!avail; + + if (copy_to_user(&args->uade[args->copied], &ade, sizeof(ade))) + return -EFAULT; + + args->copied++; + return 0; +} + +static long scoutfs_ioc_alloc_detail(struct file *file, unsigned long arg) +{ + struct super_block *sb = file_inode(file)->i_sb; + struct scoutfs_ioctl_alloc_detail __user *uad = (void __user *)arg; + struct scoutfs_ioctl_alloc_detail ad; + struct copy_alloc_detail_args args; + + if (copy_from_user(&ad, uad, sizeof(ad))) + return -EFAULT; + + args.uade = (struct scoutfs_ioctl_alloc_detail_entry __user *) + (uintptr_t)ad.entries_ptr; + args.nr = ad.entries_nr; + args.copied = 0; + + return scoutfs_alloc_foreach(sb, copy_alloc_detail_to_user, &args) ?: + args.copied; +} + +long scoutfs_ioctl(struct file *file, unsigned int cmd, unsigned long arg) +{ + switch (cmd) { + case SCOUTFS_IOC_WALK_INODES: + return scoutfs_ioc_walk_inodes(file, arg); + case SCOUTFS_IOC_INO_PATH: + return scoutfs_ioc_ino_path(file, arg); + case SCOUTFS_IOC_RELEASE: + return scoutfs_ioc_release(file, arg); + case SCOUTFS_IOC_STAGE: + return scoutfs_ioc_stage(file, arg); + case SCOUTFS_IOC_STAT_MORE: + return scoutfs_ioc_stat_more(file, arg); + case SCOUTFS_IOC_DATA_WAITING: + return scoutfs_ioc_data_waiting(file, arg); + case SCOUTFS_IOC_SETATTR_MORE: + return scoutfs_ioc_setattr_more(file, arg); + case SCOUTFS_IOC_LISTXATTR_HIDDEN: + return scoutfs_ioc_listxattr_hidden(file, arg); + case SCOUTFS_IOC_SEARCH_XATTRS: + return scoutfs_ioc_search_xattrs(file, arg); + case SCOUTFS_IOC_STATFS_MORE: + return scoutfs_ioc_statfs_more(file, arg); + case SCOUTFS_IOC_DATA_WAIT_ERR: + return scoutfs_ioc_data_wait_err(file, arg); + case SCOUTFS_IOC_ALLOC_DETAIL: + return scoutfs_ioc_alloc_detail(file, arg); + } + + return -ENOTTY; +} diff --git a/kmod/src/ioctl.h b/kmod/src/ioctl.h new file mode 100644 index 00000000..a53626a0 --- /dev/null +++ b/kmod/src/ioctl.h @@ -0,0 +1,416 @@ +#ifndef _SCOUTFS_IOCTL_H_ +#define _SCOUTFS_IOCTL_H_ + +/* + * We naturally align explicit width fields in the ioctl structs so that + * userspace doesn't need to deal with padding or unaligned packing and + * we don't have to deal with 32/64 compat. It makes it a little + * awkward to communicate persistent packed structs through the ioctls + * but that happens very rarely. An interesting special case are + * 0length arrays that follow the structs. We make those start at the + * next aligned offset of the struct to be safe. + * + * This is enforced by pahole scripting in external build environments. + */ + +/* XXX I have no idea how these are chosen. */ +#define SCOUTFS_IOCTL_MAGIC 's' + +/* + * Packed scoutfs keys rarely cross the ioctl boundary so we have a + * translation struct. + */ +struct scoutfs_ioctl_key { + __le64 _sk_first; + __le64 _sk_second; + __le64 _sk_third; + __u8 _sk_fourth; + __u8 sk_type; + __u8 sk_zone; + __u8 _pad[5]; +}; + +struct scoutfs_ioctl_walk_inodes_entry { + __u64 major; + __u64 ino; + __u32 minor; + __u8 _pad[4]; +}; + +/* + * Walk inodes in an index that is sorted by one of their fields. + * + * Each index is built from generic index items that have major and + * minor values that are set to the field being indexed. In time + * indices, for example, major is seconds and minor is nanoseconds. + * + * @first The first index entry that can be returned. + * @last The last index entry that can be returned. + * @entries_ptr Pointer to emory containing buffer for entry results. + * @nr_entries The number of entries that can fit in the buffer. + * @index Which index to walk, enumerated in _WALK_INODES_ constants. + * + * To start iterating first can be memset to 0 and last to 0xff. Then + * after each set of results first can be set to the last entry returned + * and then the fields can be incremented in reverse sort order (ino < + * minor < major) as each increasingly significant value wraps around to + * 0. + * + * These indexes are not strictly consistent. The items that back these + * index entries aren't updated with cluster locks so they're not + * guaranteed to be visible the moment you read after writing. They're + * only visible when the transaction that updated them is synced. + * + * In addition, the seq indexes will only allow walking through sequence + * space that has been consistent. This prevents old dirty entries from + * becoming visible after newer stable entries are displayed. + * + * If first is greater than last then the walk will return 0 entries. + * + * XXX invalidate before reading. + */ +struct scoutfs_ioctl_walk_inodes { + struct scoutfs_ioctl_walk_inodes_entry first; + struct scoutfs_ioctl_walk_inodes_entry last; + __u64 entries_ptr; + __u32 nr_entries; + __u8 index; + __u8 _pad[11]; /* padded to align walk_inodes_entry total size */ +}; + +enum scoutfs_ino_walk_seq_type { + SCOUTFS_IOC_WALK_INODES_META_SEQ = 0, + SCOUTFS_IOC_WALK_INODES_DATA_SEQ, + SCOUTFS_IOC_WALK_INODES_UNKNOWN, +}; + +/* + * Adds entries to the user's buffer for each inode that is found in the + * given index between the first and last positions. + */ +#define SCOUTFS_IOC_WALK_INODES _IOR(SCOUTFS_IOCTL_MAGIC, 1, \ + struct scoutfs_ioctl_walk_inodes) + +/* + * Fill the result buffer with the next absolute path to the target + * inode searching from a given position in a parent directory. + * + * @ino: The target ino that we're finding paths to. Constant across + * all the calls that make up an iteration over all the inode's paths. + * + * @dir_ino: The inode number of the directory containing the entry to + * our inode to search from. If this parent directory contains no more + * entries to our inode then we'll search through other parent directory + * inodes in inode order. + * + * @dir_pos: The position in the dir_ino parent directory of the entry + * to our inode to search from. If there is no entry at this position + * then we'll search through other entry positions in increasing order. + * If we exhaust the parent directory then we'll search through + * additional parent directories in inode order. + * + * @result_ptr: A pointer to the buffer where the result struct and + * absolute path will be stored. + * + * @result_bytes: The size of the buffer that will contain the result + * struct and the null terminated absolute path name. + * + * To start iterating set the desired target inode, dir_ino to 0, + * dir_pos to 0, and set result_ptr and _bytes to a sufficiently large + * buffeer (sizeof(result) + PATH_MAX is a solid choice). + * + * After each returned result set the next search dir_ino and dir_pos to + * the returned dir_ino and dir_pos. Then increment the search dir_pos, + * and if it wrapped to 0, increment dir_ino. + * + * This only walks back through full hard links. None of the returned + * paths will reflect symlinks to components in the path. + * + * This doesn't ensure that the caller has permissions to traverse the + * returned paths to the inode. It requires CAP_DAC_READ_SEARCH which + * bypasses permissions checking. + * + * This call is not serialized with any modification (create, rename, + * unlink) of the path components. It will return all the paths that + * were stable both before and after the call. It may or may not return + * paths which are created or unlinked during the call. + * + * On success 0 is returned and result struct is filled with the next + * absolute path. The path_bytes length of the path includes a null + * terminating byte. dir_ino and dir_pos refer to the position of the + * final component in its parent directory and can be advanced to search + * for the next terminal entry whose path is then built by walking up + * parent directories. + * + * ENOENT is returned when no paths are found. + * + * ENAMETOOLONG is returned when the result struct and path found + * doesn't fit in the result buffer. + * + * Many other errnos indicate hard failure to find the next path. + */ +struct scoutfs_ioctl_ino_path { + __u64 ino; + __u64 dir_ino; + __u64 dir_pos; + __u64 result_ptr; + __u16 result_bytes; + __u8 _pad[6]; +}; + +struct scoutfs_ioctl_ino_path_result { + __u64 dir_ino; + __u64 dir_pos; + __u16 path_bytes; + __u8 _pad[6]; + __u8 path[0]; +}; + +/* Get a single path from the root to the given inode number */ +#define SCOUTFS_IOC_INO_PATH _IOR(SCOUTFS_IOCTL_MAGIC, 2, \ + struct scoutfs_ioctl_ino_path) + +/* + * "Release" a contiguous range of logical blocks of file data. + * Released blocks are removed from the file system like truncation, but + * an offline record is left behind to trigger demand staging if the + * file is read. + * + * The starting block offset and number of blocks to release are in + * units 4KB blocks. + * + * The specified range can extend past i_size and can straddle sparse + * regions or blocks that are already offline. The only change it makes + * is to free and mark offline any existing blocks that intersect with + * the region. + * + * Returns 0 if the operation succeeds. If an error is returned then + * some partial region of the blocks in the region may have been marked + * offline. + * + * If the operation succeeds then inode metadata that reflects file data + * contents are not updated. This is intended to be transparent to the + * presentation of the data in the file. + */ +struct scoutfs_ioctl_release { + __u64 block; + __u64 count; + __u64 data_version; +}; + +#define SCOUTFS_IOC_RELEASE _IOW(SCOUTFS_IOCTL_MAGIC, 3, \ + struct scoutfs_ioctl_release) + +struct scoutfs_ioctl_stage { + __u64 data_version; + __u64 buf_ptr; + __u64 offset; + __s32 count; + __u32 _pad; +}; + +#define SCOUTFS_IOC_STAGE _IOW(SCOUTFS_IOCTL_MAGIC, 4, \ + struct scoutfs_ioctl_stage) + +/* + * Give the user inode fields that are not otherwise visible. statx() + * isn't always available and xattrs are relatively expensive. + * + * @valid_bytes stores the number of bytes that are valid in the + * structure. The caller sets this to the size of the struct that they + * understand. The kernel then fills and copies back the min of the + * size they and the user caller understand. The user can tell if a + * field is set if all of its bytes are within the valid_bytes that the + * kernel set on return. + * + * New fields are only added to the end of the struct. + */ +struct scoutfs_ioctl_stat_more { + __u64 valid_bytes; + __u64 meta_seq; + __u64 data_seq; + __u64 data_version; + __u64 online_blocks; + __u64 offline_blocks; +}; + +#define SCOUTFS_IOC_STAT_MORE _IOR(SCOUTFS_IOCTL_MAGIC, 5, \ + struct scoutfs_ioctl_stat_more) + + +struct scoutfs_ioctl_data_waiting_entry { + __u64 ino; + __u64 iblock; + __u8 op; + __u8 _pad[7]; +}; + +#define SCOUTFS_IOC_DWO_READ (1 << 0) +#define SCOUTFS_IOC_DWO_WRITE (1 << 1) +#define SCOUTFS_IOC_DWO_CHANGE_SIZE (1 << 2) +#define SCOUTFS_IOC_DWO_UNKNOWN (U8_MAX << 3) + +struct scoutfs_ioctl_data_waiting { + __u64 flags; + __u64 after_ino; + __u64 after_iblock; + __u64 ents_ptr; + __u16 ents_nr; + __u8 _pad[6]; +}; + +#define SCOUTFS_IOC_DATA_WAITING_FLAGS_UNKNOWN (U8_MAX << 0) + +#define SCOUTFS_IOC_DATA_WAITING _IOR(SCOUTFS_IOCTL_MAGIC, 6, \ + struct scoutfs_ioctl_data_waiting) + +/* + * If i_size is set then data_version must be non-zero. If the offline + * flag is set then i_size must be set and a offline extent will be + * created from offset 0 to i_size. + */ +struct scoutfs_ioctl_setattr_more { + __u64 data_version; + __u64 i_size; + __u64 flags; + __u64 ctime_sec; + __u32 ctime_nsec; + __u8 _pad[4]; +}; + +#define SCOUTFS_IOC_SETATTR_MORE_OFFLINE (1 << 0) +#define SCOUTFS_IOC_SETATTR_MORE_UNKNOWN (U8_MAX << 1) + +#define SCOUTFS_IOC_SETATTR_MORE _IOW(SCOUTFS_IOCTL_MAGIC, 7, \ + struct scoutfs_ioctl_setattr_more) + +struct scoutfs_ioctl_listxattr_hidden { + __u64 id_pos; + __u64 buf_ptr; + __u32 buf_bytes; + __u32 hash_pos; +}; + +#define SCOUTFS_IOC_LISTXATTR_HIDDEN _IOR(SCOUTFS_IOCTL_MAGIC, 8, \ + struct scoutfs_ioctl_listxattr_hidden) + +/* + * Return the inode numbers of inodes which might contain the given + * xattr. The inode may not have a set xattr with that name, the caller + * must check the returned inodes to see if they match. + * + * @next_ino: The next inode number that could be returned. Initialized + * to 0 when first searching and set to one past the last inode number + * returned to continue searching. + * @last_ino: The last inode number that could be returned. U64_MAX to + * find all inodes. + * @name_ptr: The address of the name of the xattr to search for. It is + * not null terminated. + * @inodes_ptr: The address of the array of uint64_t inode numbers in + * which to store inode numbers that may contain the xattr. EFAULT may + * be returned if this address is not naturally aligned. + * @output_flags: Set as success is returned. If an error is returned + * then this field is undefined and should not be read. + * @nr_inodes: The number of elements in the array found at inodes_ptr. + * @name_bytes: The number of non-null bytes found in the name at + * name_ptr. + * + * This requires the CAP_SYS_ADMIN capability and will return -EPERM if + * it's not granted. + * + * The number of inode numbers stored in the inodes_ptr array is + * returned. If nr_inodes is 0 or last_ino is less than next_ino then 0 + * will be immediately returned. + * + * Partial progress can be returned if an error is hit or if nr_inodes + * was larger than the internal limit on the number of inodes returned + * in a search pass. The _END output flag is set if all the results + * including last_ino were searched in this pass. + * + * It's valuable to provide a large inodes array so that all the results + * can be found in one search pass and _END can be set. There are + * significant constant costs for performing each search pass. + */ +struct scoutfs_ioctl_search_xattrs { + __u64 next_ino; + __u64 last_ino; + __u64 name_ptr; + __u64 inodes_ptr; + __u64 output_flags; + __u64 nr_inodes; + __u16 name_bytes; + __u8 _pad[6]; +}; + +/* set in output_flags if returned inodes reached last_ino */ +#define SCOUTFS_SEARCH_XATTRS_OFLAG_END (1ULL << 0) + +#define SCOUTFS_IOC_SEARCH_XATTRS _IOR(SCOUTFS_IOCTL_MAGIC, 9, \ + struct scoutfs_ioctl_search_xattrs) + +/* + * Give the user information about the filesystem. + * + * @valid_bytes stores the number of bytes that are valid in the + * structure. The caller sets this to the size of the struct that they + * understand. The kernel then fills and copies back the min of the + * size they and the user caller understand. The user can tell if a + * field is set if all of its bytes are within the valid_bytes that the + * kernel set on return. + * + * @committed_seq: All seqs up to and including this seq have been + * committed. Can be compared with meta_seq and data_seq from inodes in + * stat_more to discover if changes have been committed to disk. + * + * New fields are only added to the end of the struct. + */ +struct scoutfs_ioctl_statfs_more { + __u64 valid_bytes; + __u64 fsid; + __u64 rid; + __u64 committed_seq; + __u64 total_meta_blocks; + __u64 total_data_blocks; +}; + +#define SCOUTFS_IOC_STATFS_MORE _IOR(SCOUTFS_IOCTL_MAGIC, 10, \ + struct scoutfs_ioctl_statfs_more) + +/* + * Cause matching waiters to return an error. + * + * Find current waiters that match the inode, op, and block range to wake + * up and return an error. + */ +struct scoutfs_ioctl_data_wait_err { + __u64 ino; + __u64 data_version; + __u64 offset; + __u64 count; + __u64 op; + __s64 err; +}; + +#define SCOUTFS_IOC_DATA_WAIT_ERR _IOR(SCOUTFS_IOCTL_MAGIC, 11, \ + struct scoutfs_ioctl_data_wait_err) + + +#define SCOUTFS_IOC_ALLOC_DETAIL _IOR(SCOUTFS_IOCTL_MAGIC, 12, \ + struct scoutfs_ioctl_alloc_detail) + +struct scoutfs_ioctl_alloc_detail { + __u64 entries_ptr; + __u64 entries_nr; +}; + +struct scoutfs_ioctl_alloc_detail_entry { + __u64 id; + __u64 blocks; + __u8 type; + __u8 meta:1, + avail:1; + __u8 __bit_pad:6; + __u8 __pad[6]; +}; + +#endif diff --git a/kmod/src/item.c b/kmod/src/item.c new file mode 100644 index 00000000..2684a881 --- /dev/null +++ b/kmod/src/item.c @@ -0,0 +1,2541 @@ +/* + * Copyright (C) 2020 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "super.h" +#include "item.h" +#include "forest.h" +#include "block.h" +#include "trans.h" +#include "counters.h" +#include "scoutfs_trace.h" + +/* + * The item cache maintains a consistent view of items that are read + * from and written to the forest of btrees under the protection of + * cluster locks. + * + * The cache is built around pages of items. A page has the range of + * keys that it caches and the items that are present in that range. + * Pages are non-overlapping, there is only one page that can contain a + * given key at a time. The pages are tracked by an rbtree, and each + * page has an rbtree of items. + * + * The cache is populated by reading items from the forest of btrees + * into a private set of pages. The regions of those pages which + * weren't already cached are then inserted into the cache. + * + * CPUs can concurrently modify items that are in different pages. The + * page rbtree can be read locked to find a page, and then the page is + * locked to work with its items. We then add per-cpu references to + * recently used pages so that the global page rbtree can be skipped in + * the typical case of repeated calls to localized portions of the key + * space. + * + * Dirty items are kept in a per-page dirty list, and pages with dirty + * items are kept in a global dirty list. This reduces contention on + * the global list by accessing it at page granularity instead of every + * time an item is dirtied. The dirty items are not sorted until it + * comes time to commit them to the btrees. This reduces the cost of + * tracking dirty items during the transaction, particularly moving them + * between pages as pages are split to make room for new items. + * + * The size of the cache is only limited by memory reclaim. Pages are + * kept in a very coarse lru. Dirtying doesn't remove pages from the + * lru, and is operating against lock ordering with trylocks, so + * shrinking can rarely have to skip pages in the LRU. + * + * The locking is built around the fast path of everyone checking the + * the page rbtree, then locking pages, and then adding or removing + * pages from the lru or dirty lists. Writing and the shrinker work + * work in reverse, starting with the dirty or lru lists and have to use + * trylock to lock the pages. When we split we have to lock multiple + * pages and we use trylock which is guaranteed to succeed because the + * pages are private. + */ + +struct item_cache_info { + /* almost always read, barely written */ + struct super_block *sb; + struct item_percpu_pages __percpu *pcpu_pages; + struct shrinker shrinker; + struct notifier_block notifier; + + /* often walked, but per-cpu refs are fast path */ + rwlock_t rwlock; + struct rb_root pg_root; + + /* page-granular modification by writers, then exclusive to commit */ + spinlock_t dirty_lock; + struct list_head dirty_list; + atomic_t dirty_pages; + + /* page-granular modification by readers */ + spinlock_t lru_lock; + struct list_head lru_list; + unsigned long lru_pages; + + /* written by page readers, read by shrink */ + spinlock_t active_lock; + struct rb_root active_root; +}; + +#define DECLARE_ITEM_CACHE_INFO(sb, name) \ + struct item_cache_info *name = SCOUTFS_SB(sb)->item_cache_info + +#define PG_PER_CPU 32 +struct item_percpu_pages { + struct rb_root root; + struct list_head list; + struct pcpu_page_ref { + struct scoutfs_key start; + struct scoutfs_key end; + struct cached_page *pg; + struct rb_node node; + struct list_head head; + } refs[PG_PER_CPU]; +}; + +struct cached_page { + /* often read by concurrent rbtree walks */ + struct rb_node node; + struct scoutfs_key start; + struct scoutfs_key end; + + /* often modified by page rwlock holder */ + rwlock_t rwlock; + struct rb_root item_root; + struct list_head lru_head; + unsigned long lru_time; + struct list_head dirty_list; + struct list_head dirty_head; + struct page *page; + unsigned int page_off; + unsigned int erased_bytes; + atomic_t refcount; +}; + +struct cached_item { + struct rb_node node; + struct list_head dirty_head; + unsigned int dirty:1, /* needs to be written */ + persistent:1, /* in btrees, needs deletion item */ + deletion:1; /* negative del item for writing */ + unsigned int val_len; + struct scoutfs_key key; + struct scoutfs_log_item_value liv; + char val[0]; +}; + +#define CACHED_ITEM_ALIGN 8 + +static int item_val_bytes(int val_len) +{ + return round_up(offsetof(struct cached_item, val[val_len]), CACHED_ITEM_ALIGN); +} + +/* + * Return if the page has room to allocate an item with the given value + * length at its free page offset. This must be called with the page + * writelock held because it can modify the page to reclaim free space + * to mkae room for the allocation. Today all it does is recognize that + * the page is empty and reset the page_off. + */ +static bool page_has_room(struct cached_page *pg, int val_len) +{ + if (RB_EMPTY_ROOT(&pg->item_root)) + pg->page_off = 0; + + return pg->page_off + item_val_bytes(val_len) <= PAGE_SIZE; +} + +static struct cached_page *first_page(struct rb_root *root) +{ + struct rb_node *node; + + if (!root || !(node = rb_first(root))) + return NULL; + + return rb_entry(node, struct cached_page, node); +} + +static struct cached_item *first_item(struct rb_root *root) +{ + struct rb_node *node; + + if (!root || !(node = rb_first(root))) + return NULL; + + return rb_entry(node, struct cached_item, node); +} + +static struct cached_item *last_item(struct rb_root *root) +{ + struct rb_node *node; + + if (!root || !(node = rb_last(root))) + return NULL; + + return rb_entry(node, struct cached_item, node); +} + +static struct cached_item *next_item(struct cached_item *item) +{ + struct rb_node *node; + + if (!item || !(node = rb_next(&item->node))) + return NULL; + + return rb_entry(node, struct cached_item, node); +} + +static struct cached_item *prev_item(struct cached_item *item) +{ + struct rb_node *node; + + if (!item || !(node = rb_prev(&item->node))) + return NULL; + + return rb_entry(node, struct cached_item, node); +} + +static void rbtree_insert(struct rb_node *node, struct rb_node *par, + struct rb_node **pnode, struct rb_root *root) +{ + rb_link_node(node, par, pnode); + rb_insert_color(node, root); +} + +static void rbtree_erase(struct rb_node *node, struct rb_root *root) +{ + rb_erase(node, root); + RB_CLEAR_NODE(node); +} + +static void rbtree_replace_node(struct rb_node *victim, struct rb_node *new, + struct rb_root *root) +{ + rb_replace_node(victim, new, root); + RB_CLEAR_NODE(victim); +} + +/* + * This is far too expensive to use regularly, but it's very helpful for + * discovering corruption after modifications to cached pages. + */ +static __attribute__((unused)) void verify_page_rbtree(struct rb_root *root) +{ + struct cached_item *item; + struct cached_page *par; + struct cached_page *pg; + struct cached_page *n; + char *reason = NULL; + struct rb_node *p; + int cmp; + + rbtree_postorder_for_each_entry_safe(pg, n, root, node) { + + item = NULL; + par = NULL; + + if (scoutfs_key_compare(&pg->start, &pg->end) > 0) { + reason = "start > end"; + break; + } + + item = first_item(&pg->item_root); + if (item && scoutfs_key_compare(&item->key, &pg->start) < 0) { + reason = "first item < start"; + break; + } + + item = last_item(&pg->item_root); + if (item && scoutfs_key_compare(&item->key, &pg->end) > 0) { + reason = "last item > end"; + break; + } + + p = rb_parent(&pg->node); + if (!p) + continue; + par = rb_entry(p, struct cached_page, node); + + cmp = scoutfs_key_compare_ranges(&pg->start, &pg->end, + &par->start, &par->end); + if (cmp == 0) { + reason = "parent and child overlap"; + break; + } + + if (par->node.rb_right == &pg->node && cmp < 0) { + reason = "right child < parent"; + break; + } + + if (par->node.rb_left == &pg->node && cmp > 0) { + reason = "left child > parent"; + break; + } + } + + if (!reason) + return; + + printk("bad item page rbtree: %s\n", reason); + printk("pg %p start "SK_FMT" end "SK_FMT"\n", + pg, SK_ARG(&pg->start), SK_ARG(&pg->end)); + if (par) + printk("par %p start "SK_FMT" end "SK_FMT"\n", + par, SK_ARG(&par->start), SK_ARG(&par->end)); + if (item) + printk("item %p key "SK_FMT"\n", item, SK_ARG(&item->key)); + + rbtree_postorder_for_each_entry_safe(pg, n, root, node) { + printk(" pg %p left %p right %p start "SK_FMT" end "SK_FMT"\n", + pg, + pg->node.rb_left ? rb_entry(pg->node.rb_left, + struct cached_page, node) : + NULL, + pg->node.rb_right ? rb_entry(pg->node.rb_right, + struct cached_page, node) : + NULL, + SK_ARG(&pg->start), + SK_ARG(&pg->end)); + } + + BUG(); +} + + +/* + * This lets us lock newly allocated pages without having to add nesting + * annotation. The non-acquired path is never executed. + */ +static void write_trylock_will_succeed(rwlock_t *rwlock) +__acquires(rwlock) +{ + while (!write_trylock(rwlock)) + BUG(); +} + +static struct cached_page *alloc_pg(struct super_block *sb, gfp_t gfp) +{ + struct cached_page *pg; + struct page *page; + + pg = kzalloc(sizeof(struct cached_page), GFP_NOFS | gfp); + page = alloc_page(GFP_NOFS | gfp); + if (!page || !pg) { + kfree(pg); + __free_page(page); + return NULL; + } + + scoutfs_inc_counter(sb, item_page_alloc); + + RB_CLEAR_NODE(&pg->node); + rwlock_init(&pg->rwlock); + pg->item_root = RB_ROOT; + INIT_LIST_HEAD(&pg->lru_head); + INIT_LIST_HEAD(&pg->dirty_list); + INIT_LIST_HEAD(&pg->dirty_head); + pg->page = page; + atomic_set(&pg->refcount, 1); + + return pg; +} + +static void get_pg(struct cached_page *pg) +{ + atomic_inc(&pg->refcount); +} + +static void put_pg(struct super_block *sb, struct cached_page *pg) +{ + if (pg && atomic_dec_and_test(&pg->refcount)) { + scoutfs_inc_counter(sb, item_page_free); + + BUG_ON(!RB_EMPTY_NODE(&pg->node)); + BUG_ON(!list_empty(&pg->lru_head)); + BUG_ON(!list_empty(&pg->dirty_list)); + BUG_ON(!list_empty(&pg->dirty_head)); + + __free_page(pg->page); + kfree(pg); + } +} + +/* + * Allocate space for a new item from the free offset at the end of a + * cached page. This isn't a blocking allocation, and it's likely that + * the caller has ensured it will succeed by allocating from a new empty + * page or checking the free space first. + */ +static struct cached_item *alloc_item(struct cached_page *pg, + struct scoutfs_key *key, + struct scoutfs_log_item_value *liv, + void *val, int val_len) +{ + struct cached_item *item; + + if (!page_has_room(pg, val_len)) + return NULL; + + item = page_address(pg->page) + pg->page_off; + pg->page_off += item_val_bytes(val_len); + + RB_CLEAR_NODE(&item->node); + INIT_LIST_HEAD(&item->dirty_head); + item->dirty = 0; + item->persistent = 0; + item->deletion = !!(liv->flags & SCOUTFS_LOG_ITEM_FLAG_DELETION); + item->val_len = val_len; + item->key = *key; + item->liv = *liv; + + if (val_len) + memcpy(item->val, val, val_len); + + return item; +} + +static void erase_item(struct cached_page *pg, struct cached_item *item) +{ + rbtree_erase(&item->node, &pg->item_root); + pg->erased_bytes += round_up(item_val_bytes(item->val_len), + CACHED_ITEM_ALIGN); +} + +static void lru_add(struct super_block *sb, struct item_cache_info *cinf, + struct cached_page *pg) +{ + spin_lock(&cinf->lru_lock); + if (list_empty(&pg->lru_head)) { + scoutfs_inc_counter(sb, item_page_lru_add); + list_add_tail(&pg->lru_head, &cinf->lru_list); + cinf->lru_pages++; + } + spin_unlock(&cinf->lru_lock); +} + +static void __lru_remove(struct super_block *sb, struct item_cache_info *cinf, + struct cached_page *pg) +{ + if (!list_empty(&pg->lru_head)) { + scoutfs_inc_counter(sb, item_page_lru_remove); + list_del_init(&pg->lru_head); + cinf->lru_pages--; + } +} + +static void lru_remove(struct super_block *sb, struct item_cache_info *cinf, + struct cached_page *pg) +{ + spin_lock(&cinf->lru_lock); + __lru_remove(sb, cinf, pg); + spin_unlock(&cinf->lru_lock); +} + +/* + * Make sure that the page the caller just accessed is reasonably close + * to the tail of the lru so it will be less likely to be reclaimed by + * the shrinker. + * + * We want to quickly determine that the page is close enough to the + * tail by only looking at the page. We use a coarse clock tick to + * determine if we've already moved the head to the tail sufficiently + * recently. We can't differentiate shrinking priority amongst the + * number of pages that the cpu can access within given chunk of time. + * + * We don't care that the lru_time accessed aren't locked and could see + * rare corruption. It's just a shrink priority heuristic. + */ +static void lru_accessed(struct super_block *sb, struct item_cache_info *cinf, + struct cached_page *pg) +{ + unsigned long time = jiffies_to_msecs(jiffies); + + scoutfs_inc_counter(sb, item_page_accessed); + + if (pg->lru_time != time) { + lru_remove(sb, cinf, pg); + pg->lru_time = time; + lru_add(sb, cinf, pg); + } +} + +/* + * Return the pg that contains the key and set the parent nodes for insertion. + * When we find the pg we go right so that the caller can insert a new + * page to the right of the found page if it had to split the page. + */ +static struct cached_page *page_rbtree_walk(struct super_block *sb, + struct rb_root *root, + struct scoutfs_key *start, + struct scoutfs_key *end, + struct cached_page **prev, + struct cached_page **next, + struct rb_node **par, + struct rb_node ***pnode) +{ + struct rb_node **node = &root->rb_node; + struct rb_node *parent = NULL; + struct cached_page *ret = NULL; + struct cached_page *pg; + int cmp; + + scoutfs_inc_counter(sb, item_page_rbtree_walk); + + if (next) + *next = NULL; + if (prev) + *prev = NULL; + + while (*node) { + parent = *node; + pg = container_of(*node, struct cached_page, node); + + cmp = scoutfs_key_compare_ranges(start, end, &pg->start, + &pg->end); + if (cmp < 0) { + if (next) + *next = pg; + node = &(*node)->rb_left; + } else if (cmp > 0) { + if (prev) + *prev = pg; + node = &(*node)->rb_right; + } else { + ret = pg; + node = &(*node)->rb_right; + } + } + + if (par) + *par = parent; + if (pnode) + *pnode = node; + + return ret; +} + +#define for_each_page_safe(root, pg, tmp) \ + for (tmp = rb_first(root); \ + tmp && (pg = container_of(tmp, struct cached_page, node)) && \ + ((tmp = rb_next(tmp)), 1); ) + +static struct cached_item *item_rbtree_walk(struct rb_root *root, + struct scoutfs_key *key, + struct cached_item **next, + struct rb_node **par, + struct rb_node ***pnode) +{ + struct rb_node **node = &root->rb_node; + struct rb_node *parent = NULL; + struct cached_item *ret = NULL; + struct cached_item *item; + int cmp; + + if (next) + *next = NULL; + + while (*node) { + parent = *node; + item = container_of(*node, struct cached_item, node); + + cmp = scoutfs_key_compare(key, &item->key); + if (cmp < 0) { + if (next) + *next = item; + node = &(*node)->rb_left; + } else if (cmp > 0) { + node = &(*node)->rb_right; + } else { + ret = item; + node = &(*node)->rb_left; + } + } + + if (par) + *par = parent; + if (pnode) + *pnode = node; + + return ret; +} + +#define for_each_item_from_safe(root, item, tmp, key) \ + for (item = item_rbtree_walk(root, key, &tmp, NULL, NULL) ?: tmp; \ + item && ((tmp = next_item(item)), 1); \ + item = tmp) + +#define for_each_item_safe(root, item, tmp) \ + for (tmp = rb_first(root); \ + tmp && (item = container_of(tmp, struct cached_item, node)) && \ + ((tmp = rb_next(tmp)), 1); ) + +/* + * As we mark the first and clear the last items in a page, we add or + * delete the page from the dirty list. The caller can give us a page + * to add the newly dirtied page after, rather than at the tail of the + * list. + */ +static void mark_item_dirty(struct super_block *sb, + struct item_cache_info *cinf, + struct cached_page *pg, + struct cached_page *after, + struct cached_item *item) +{ + if (!item->dirty) { + if (list_empty(&pg->dirty_list)) { + scoutfs_inc_counter(sb, item_page_mark_dirty); + spin_lock(&cinf->dirty_lock); + if (after) + list_add(&pg->dirty_head, &after->dirty_head); + else + list_add_tail(&pg->dirty_head, + &cinf->dirty_list); + atomic_inc(&cinf->dirty_pages); + spin_unlock(&cinf->dirty_lock); + } + + scoutfs_inc_counter(sb, item_mark_dirty); + list_add_tail(&item->dirty_head, &pg->dirty_list); + item->dirty = 1; + } +} + +static void clear_item_dirty(struct super_block *sb, + struct item_cache_info *cinf, + struct cached_page *pg, + struct cached_item *item) +{ + if (item->dirty) { + scoutfs_inc_counter(sb, item_clear_dirty); + item->dirty = 0; + list_del_init(&item->dirty_head); + + if (list_empty(&pg->dirty_list)) { + scoutfs_inc_counter(sb, item_page_clear_dirty); + spin_lock(&cinf->dirty_lock); + list_del_init(&pg->dirty_head); + atomic_dec(&cinf->dirty_pages); + spin_unlock(&cinf->dirty_lock); + } + } +} + +static void erase_page_items(struct cached_page *pg, + struct scoutfs_key *start, + struct scoutfs_key *end) +{ + struct cached_item *item; + struct cached_item *tmp; + + for_each_item_from_safe(&pg->item_root, item, tmp, start) { + + /* only called in unused read regions or read_pages pages */ + BUG_ON(item->dirty); + + if (scoutfs_key_compare(&item->key, end) > 0) + break; + + erase_item(pg, item); + } +} + +/* + * Move all the items starting from the key and stopping before moving + * the stop key. The right destination page must be empty. Items are + * copied in tree order which lets us easily insert after each previous + * item. + * + * This preserves dirty page and item ordering by adding the right page + * to the dirty list after the left page, and by adding items to the + * tail of right's dirty list in key sort order. + * + * The caller is responsible for page locking and managing the lru. + */ +static void move_page_items(struct super_block *sb, + struct item_cache_info *cinf, + struct cached_page *left, + struct cached_page *right, + struct scoutfs_key *key, + struct scoutfs_key *stop) +{ + struct cached_item *from; + struct cached_item *to; + struct cached_item *tmp; + struct rb_node **pnode; + struct rb_node *par; + + /* really empty right destination? */ + BUG_ON(!RB_EMPTY_ROOT(&right->item_root)); + par = NULL; + pnode = &right->item_root.rb_node; + + for_each_item_from_safe(&left->item_root, from, tmp, key) { + + if (stop && scoutfs_key_compare(&from->key, stop) >= 0) + break; + + to = alloc_item(right, &from->key, &from->liv, from->val, + from->val_len); + rbtree_insert(&to->node, par, pnode, &right->item_root); + par = &to->node; + pnode = &to->node.rb_right; + + if (from->dirty) { + mark_item_dirty(sb, cinf, right, left, to); + clear_item_dirty(sb, cinf, left, from); + } + + to->persistent = from->persistent; + to->deletion = from->deletion; + + erase_item(left, from); + } +} + +enum page_intersection_type { + PGI_DISJOINT, + PGI_INSIDE, + PGI_START_OLAP, + PGI_END_OLAP, + PGI_BISECT_NEEDED, + PGI_BISECT, +}; + +/* + * Remove items from the page with intersect with the range. We return + * a code to indicate which kind of intersection occurred. The caller + * provides the right page to move items to if the page is bisected by + * the range. + * + * This modifies the page keys so it needs to be held with a write page + * rbtree lock if the page is in the page rbtree. + */ +static int trim_page_intersection(struct super_block *sb, + struct item_cache_info *cinf, + struct cached_page *pg, + struct cached_page *right, + struct scoutfs_key *start, + struct scoutfs_key *end) +{ + int ps_e = scoutfs_key_compare(&pg->start, end); + int pe_s = scoutfs_key_compare(&pg->end, start); + int ps_s; + int pe_e; + + /* + * page and range don't intersect + * + * ps |----------| pe + * s |----------| e + * (or) + * ps |----------| pe + * s |----------| e + */ + if (ps_e > 0 || pe_s < 0) + return PGI_DISJOINT; + + ps_s = scoutfs_key_compare(&pg->start, start); + pe_e = scoutfs_key_compare(&pg->end, end); + + /* + * page entirely inside range + * + * ps |----------| pe + * s |----------| e + */ + if (ps_s >= 0 && pe_e <= 0) + return PGI_INSIDE; + + /* + * page surrounds range, and is bisected by it + * + * ps |----------| pe + * s |------| e + */ + if (ps_s < 0 && pe_e > 0) { + if (!right) + return PGI_BISECT_NEEDED; + + right->start = *end; + scoutfs_key_inc(&right->start); + right->end = pg->end; + pg->end = *start; + scoutfs_key_dec(&pg->end); + erase_page_items(pg, start, end); + move_page_items(sb, cinf, pg, right, &right->start, NULL); + return PGI_BISECT; + } + + /* + * start of page overlaps with range + * + * ps |----------| pe + * s |----------| e + */ + if (pe_e > 0) { + /* start of page overlaps range */ + pg->start = *end; + scoutfs_key_inc(&pg->start); + erase_page_items(pg, start, end); + return PGI_START_OLAP; + } + + /* + * end of page overlaps with range + * + * ps |----------| pe + * s |----------| e + */ + pg->end = *start; + scoutfs_key_dec(&pg->end); + erase_page_items(pg, start, end); + return PGI_END_OLAP; +} + +/* + * The caller wants to allocate an item in the page but there isn't room + * at the page_off. If erasing items has left sufficient internal free + * space we can pack the existing items to the start of the page to make + * room for the insertion. + * + * The caller's empty pg is only used for its page struct, which we swap + * with our old empty page. We don't touch its pg struct. + * + * This is a coarse bulk way of dealing with free space, as opposed to + * specifically tracking internal free regions and using them to satisfy + * item allocations. + */ +static void compact_page_items(struct super_block *sb, + struct cached_page *pg, + struct cached_page *empty) +{ + struct cached_item *from; + struct cached_item *to; + struct rb_root item_root = RB_ROOT; + struct rb_node *par = NULL; + struct rb_node **pnode = &item_root.rb_node; + unsigned int page_off = 0; + LIST_HEAD(dirty_list); + + if (pg->erased_bytes < item_val_bytes(SCOUTFS_MAX_VAL_SIZE)) + return; + + if (WARN_ON_ONCE(empty->page_off != 0) || + WARN_ON_ONCE(!RB_EMPTY_ROOT(&empty->item_root)) || + WARN_ON_ONCE(!list_empty(&empty->dirty_list))) + return; + + scoutfs_inc_counter(sb, item_page_compact); + + for (from = first_item(&pg->item_root); from; from = next_item(from)) { + to = page_address(empty->page) + page_off; + page_off += round_up(item_val_bytes(from->val_len), + CACHED_ITEM_ALIGN); + + /* copy the entire item, struct members and all */ + memcpy(to, from, item_val_bytes(from->val_len)); + + rbtree_insert(&to->node, par, pnode, &item_root); + par = &to->node; + pnode = &to->node.rb_right; + + if (to->dirty) + list_add_tail(&to->dirty_head, &dirty_list); + } + + pg->item_root = item_root; + list_replace(&dirty_list, &pg->dirty_list); + swap(pg->page, empty->page); + pg->page_off = page_off; + pg->erased_bytes = 0; +} + +/* + * This behaves a little differently than the other walks because we + * want to minimize compares and there are only simple searching and + * inserting callers. + */ +static struct pcpu_page_ref *pcpu_page_rbtree_walk(struct rb_root *root, + struct scoutfs_key *key, + struct pcpu_page_ref *ins) +{ + struct rb_node **node = &root->rb_node; + struct rb_node *parent = NULL; + struct pcpu_page_ref *ret = NULL; + struct pcpu_page_ref *ref; + int cmp; + + while (*node) { + parent = *node; + ref = container_of(*node, struct pcpu_page_ref, node); + + cmp = scoutfs_key_compare_ranges(key, key, + &ref->start, &ref->end); + if (cmp < 0) { + node = &(*node)->rb_left; + } else if (cmp > 0) { + node = &(*node)->rb_right; + } else { + ret = ref; + if (!ins) + return ret; + node = &(*node)->rb_right; + } + } + + if (ins) + rbtree_insert(&ins->node, parent, node, root); + + return ret; +} + +/* + * Search the per-cpu page references for a page that contains the key + * the caller needs. These lookups are very frequent and key + * comparisons are relatively expensive, so we use an rbtree to decrease + * the comparison costs, particularly of misses. + * + * All the references in all the cpus go stale as page key boundaries + * are modified by reading, insertion, and invalidation. If we find a + * stale ref we will drop it, but otherwise we let stale refs age out as + * new refs are inserted. + */ +static struct cached_page *get_pcpu_page(struct super_block *sb, + struct item_cache_info *cinf, + struct scoutfs_key *key, + bool write) +{ + struct item_percpu_pages *pages = get_cpu_ptr(cinf->pcpu_pages); + struct cached_page *pg = NULL; + struct pcpu_page_ref *ref; + + ref = pcpu_page_rbtree_walk(&pages->root, key, NULL); + if (ref) { + pg = ref->pg; + if (write) + write_lock(&pg->rwlock); + else + read_lock(&pg->rwlock); + + if (scoutfs_key_compare_ranges(key, key, + &pg->start, &pg->end)) { + if (write) + write_unlock(&pg->rwlock); + else + read_unlock(&pg->rwlock); + + scoutfs_inc_counter(sb, item_pcpu_page_miss_keys); + rbtree_erase(&ref->node, &pages->root); + list_move_tail(&ref->head, &pages->list); + put_pg(sb, pg); + ref->pg = NULL; + pg = NULL; + } else { + if (pages->list.next != &ref->head) + list_move(&ref->head, &pages->list); + __release(pg_rwlock); + } + } + + put_cpu_ptr(cinf->pcpu_pages); + + if (pg) + scoutfs_inc_counter(sb, item_pcpu_page_hit); + else + scoutfs_inc_counter(sb, item_pcpu_page_miss); + + return pg; +} + +/* + * The caller has a locked page that it knows is authoritative for its + * range of keys. Add it to this cpu's cache and remove any other page + * in the pool which intersects with its range. + */ +static void add_pcpu_page(struct super_block *sb, struct item_cache_info *cinf, + struct cached_page *pg) +{ + struct item_percpu_pages *pages = get_cpu_ptr(cinf->pcpu_pages); + struct pcpu_page_ref *old; + struct pcpu_page_ref *ref; + + ref = list_last_entry(&pages->list, struct pcpu_page_ref, head); + if (ref->pg) { + rbtree_erase(&ref->node, &pages->root); + put_pg(sb, ref->pg); + } + ref->start = pg->start; + ref->end = pg->end; + ref->pg = pg; + get_pg(pg); + + list_move(&ref->head, &pages->list); + + old = pcpu_page_rbtree_walk(&pages->root, &ref->end, ref); + if (old) { + scoutfs_inc_counter(sb, item_pcpu_add_replaced); + rbtree_erase(&old->node, &pages->root); + list_move_tail(&old->head, &pages->list); + put_pg(sb, old->pg); + old->pg = NULL; + } + + put_cpu_ptr(cinf->pcpu_pages); +} + +/* + * If a page is removed from the page rbtree we clear its keys so that percpu + * references won't use the page and will drop their reference. Must be + * called with a write page rwlock. + */ +static void invalidate_pcpu_page(struct cached_page *pg) +{ + scoutfs_key_set_zeros(&pg->start); + scoutfs_key_set_zeros(&pg->end); +} + +static void init_pcpu_pages(struct item_cache_info *cinf, int cpu) +{ + struct item_percpu_pages *pages = per_cpu_ptr(cinf->pcpu_pages, cpu); + struct pcpu_page_ref *ref; + int i; + + pages->root = RB_ROOT; + INIT_LIST_HEAD(&pages->list); + + for (i = 0; i < ARRAY_SIZE(pages->refs); i++) { + ref = &pages->refs[i]; + + ref->pg = NULL; + list_add_tail(&ref->head, &pages->list); + } +} + +static void drop_pcpu_pages(struct super_block *sb, + struct item_cache_info *cinf, int cpu) +{ + struct item_percpu_pages *pages = per_cpu_ptr(cinf->pcpu_pages, cpu); + struct pcpu_page_ref *ref; + int i; + + for (i = 0; i < ARRAY_SIZE(pages->refs); i++) { + ref = &pages->refs[i]; + + if (ref->pg) + put_pg(sb, ref->pg); + ref->pg = NULL; + } + + pages->root = RB_ROOT; +} + +/* + * Set the keys of the destination pages of a split. We try to find the + * key which balances the space consumed by items in the resulting split + * pages. We move the split key to the right, setting the left end by + * decrementing that key. We bias towards advancing the left item first + * so that we don't use it and possibly decrementing the starting page + * key. We can't have a page that covers a single key. Callers of + * split should have tried compacting which ensures that if we split we + * must have multiple items, even if they all have the max value length. + */ +static void set_split_keys(struct cached_page *pg, struct cached_page *left, + struct cached_page *right) +{ + struct cached_item *left_item = first_item(&pg->item_root); + struct cached_item *right_item = last_item(&pg->item_root); + struct cached_item *mid; + int left_tot = 0; + int right_tot = 0; + + BUILD_BUG_ON((PAGE_SIZE / SCOUTFS_MAX_VAL_SIZE) < 4); + BUG_ON(scoutfs_key_compare(&pg->start, &pg->end) > 0); + BUG_ON(left_item == NULL); + BUG_ON(right_item == NULL); + BUG_ON(left_item == right_item); + + while (left_item && right_item && left_item != right_item) { + if (left_tot <= right_tot) { + left_tot += item_val_bytes(left_item->val_len); + left_item = next_item(left_item); + } else { + right_tot += item_val_bytes(right_item->val_len); + right_item = prev_item(right_item); + } + } + + mid = left_item ?: right_item; + + left->start = pg->start; + left->end = mid->key; + scoutfs_key_dec(&left->end); + right->start = mid->key; + right->end = pg->end; +} + +/* + * The caller found a page that didn't have room for the item they + * wanted to allocate. We allocate pages for the split and see if the + * page still needs splitting once we've locked it. + * + * To modify page keys we need a write lock on the page rbtree, which + * globally prevents reads from finding pages. We want to minimize this + * so we add empty pages with the split ranges to the rbtree and then + * perform the item motion only with the page locks held. This will + * exclude any users of the items in the affected range. + */ +static int try_split_page(struct super_block *sb, struct item_cache_info *cinf, + struct scoutfs_key *key, int val_len) +{ + struct cached_page *right; + struct cached_page *left; + struct cached_page *pg; + struct cached_item *item; + struct rb_node **pnode; + struct rb_node *par; + int ret; + + left = alloc_pg(sb, 0); + right = alloc_pg(sb, 0); + if (!left || !right) { + ret = -ENOMEM; + goto out; + } + + write_lock(&cinf->rwlock); + + pg = page_rbtree_walk(sb, &cinf->pg_root, key, key, NULL, NULL, + &par, &pnode); + if (pg == NULL) { + write_unlock(&cinf->rwlock); + ret = 0; + goto out; + } + + write_lock(&pg->rwlock); + + if (!page_has_room(pg, val_len)) + compact_page_items(sb, pg, left); + + if (page_has_room(pg, val_len)) { + write_unlock(&cinf->rwlock); + write_unlock(&pg->rwlock); + ret = 0; + goto out; + } + + /* special case adding an empty page when key is after the last item */ + item = last_item(&pg->item_root); + if (scoutfs_key_compare(key, &item->key) > 0) { + right->start = *key; + right->end = pg->end; + pg->end = *key; + scoutfs_key_dec(&pg->end); + + write_trylock_will_succeed(&right->rwlock); + rbtree_insert(&right->node, par, pnode, &cinf->pg_root); + lru_accessed(sb, cinf, right); + + /* adding right first removes pg */ + add_pcpu_page(sb, cinf, right); + add_pcpu_page(sb, cinf, pg); + + write_unlock(&cinf->rwlock); + write_unlock(&pg->rwlock); + write_unlock(&right->rwlock); + right = NULL; + ret = 0; + goto out; + } + + scoutfs_inc_counter(sb, item_page_split); + + /* pages are still private, tylock will succeed */ + write_trylock_will_succeed(&left->rwlock); + write_trylock_will_succeed(&right->rwlock); + + set_split_keys(pg, left, right); + + rbtree_insert(&right->node, par, pnode, &cinf->pg_root); + rbtree_replace_node(&pg->node, &left->node, &cinf->pg_root); + lru_remove(sb, cinf, pg); + + write_unlock(&cinf->rwlock); + + /* move items while only holding page locks, visible once unlocked */ + move_page_items(sb, cinf, pg, left, &left->start, &right->start); + lru_accessed(sb, cinf, left); + add_pcpu_page(sb, cinf, left); + write_unlock(&left->rwlock); + left = NULL; + + move_page_items(sb, cinf, pg, right, &right->start, NULL); + lru_accessed(sb, cinf, right); + add_pcpu_page(sb, cinf, right); + write_unlock(&right->rwlock); + right = NULL; + + /* and drop the source page, it was replaced above */ + invalidate_pcpu_page(pg); + write_unlock(&pg->rwlock); + put_pg(sb, pg); + + ret = 0; +out: + put_pg(sb, left); + put_pg(sb, right); + return ret; +} + +/* + * The caller has a write-only cluster lock and wants to populate the + * cache so that it can insert an item without reading. They found a + * hole but unlocked so we check again under the lock after allocating. + * We insert an empty page that covers the key and extends to either the + * neighbours or the caller's (lock's) range. + */ +static int cache_empty_page(struct super_block *sb, + struct item_cache_info *cinf, + struct scoutfs_key *key, struct scoutfs_key *start, + struct scoutfs_key *end) +{ + struct cached_page *prev; + struct cached_page *next; + struct cached_page *pg; + struct rb_node **pnode; + struct rb_node *par; + + pg = alloc_pg(sb, 0); + if (!pg) + return -ENOMEM; + + write_lock(&cinf->rwlock); + + if (!page_rbtree_walk(sb, &cinf->pg_root, key, key, &prev, &next, + &par, &pnode)) { + pg->start = *start; + if (prev && scoutfs_key_compare(&prev->end, start) > 0) { + pg->start = prev->end; + scoutfs_key_inc(&pg->start); + } + + pg->end = *end; + if (next && scoutfs_key_compare(&next->start, end) < 0) { + pg->end = next->start; + scoutfs_key_dec(&pg->end); + } + + rbtree_insert(&pg->node, par, pnode, &cinf->pg_root); + lru_accessed(sb, cinf, pg); + pg = NULL; + } + + write_unlock(&cinf->rwlock); + + put_pg(sb, pg); + + return 0; +} + +struct active_reader { + struct rb_node node; + struct scoutfs_key start; + struct scoutfs_key end; +}; + +static struct active_reader *active_rbtree_walk(struct rb_root *root, + struct scoutfs_key *start, + struct scoutfs_key *end, + struct rb_node **par, + struct rb_node ***pnode) +{ + struct rb_node **node = &root->rb_node; + struct rb_node *parent = NULL; + struct active_reader *ret = NULL; + struct active_reader *active; + int cmp; + + while (*node) { + parent = *node; + active = container_of(*node, struct active_reader, node); + + cmp = scoutfs_key_compare_ranges(start, end, &active->start, + &active->end); + if (cmp < 0) { + node = &(*node)->rb_left; + } else if (cmp > 0) { + node = &(*node)->rb_right; + } else { + ret = active; + node = &(*node)->rb_left; + } + } + + if (par) + *par = parent; + if (pnode) + *pnode = node; + + return ret; +} + +/* + * Add a newly read item to the pages that we're assembling for + * insertion into the cache. These pages are private, they only exist + * on our root and aren't in dirty or lru lists. + * + * We need to store deletion items here as we read items from all the + * btrees so that they can override older versions of the items. The + * deletion items will be deleted before we insert the pages into the + * cache. We don't insert old versions of items into the tree here so + * that the trees don't have to compare versions. + */ +static int read_page_item(struct super_block *sb, struct scoutfs_key *key, + struct scoutfs_log_item_value *liv, void *val, + int val_len, void *arg) +{ + DECLARE_ITEM_CACHE_INFO(sb, cinf); + struct rb_root *root = arg; + struct cached_page *right = NULL; + struct cached_page *left = NULL; + struct cached_page *pg; + struct cached_item *found; + struct cached_item *item; + struct rb_node *p_par; + struct rb_node *par; + struct rb_node **p_pnode; + struct rb_node **pnode; + + pg = page_rbtree_walk(sb, root, key, key, NULL, NULL, &p_par, &p_pnode); + found = item_rbtree_walk(&pg->item_root, key, NULL, &par, &pnode); + if (found && (le64_to_cpu(found->liv.vers) >= le64_to_cpu(liv->vers))) + return 0; + + if (!page_has_room(pg, val_len)) { + left = alloc_pg(sb, 0); + /* split needs multiple items, sparse may not have enough */ + if (!left) + return -ENOMEM; + compact_page_items(sb, pg, left); + } + + item = alloc_item(pg, key, liv, val, val_len); + if (!item) { + /* simpler split of private pages, no locking/dirty/lru */ + if (!left) + left = alloc_pg(sb, 0); + right = alloc_pg(sb, 0); + if (!left || !right) { + put_pg(sb, left); + put_pg(sb, right); + return -ENOMEM; + } + + scoutfs_inc_counter(sb, item_read_pages_split); + + set_split_keys(pg, left, right); + rbtree_insert(&right->node, p_par, p_pnode, root); + rbtree_replace_node(&pg->node, &left->node, root); + move_page_items(sb, cinf, pg, left, + &left->start, &right->start); + move_page_items(sb, cinf, pg, right, &right->start, NULL); + put_pg(sb, pg); + + pg = scoutfs_key_compare(key, &left->end) <= 0 ? left : right; + item = alloc_item(pg, key, liv, val, val_len); + found = item_rbtree_walk(&pg->item_root, key, NULL, &par, + &pnode); + + left = NULL; + right = NULL; + } + + /* if deleted a deletion item will be required */ + item->persistent = 1; + + rbtree_insert(&item->node, par, pnode, &pg->item_root); + if (found) + erase_item(pg, found); + + put_pg(sb, left); + put_pg(sb, right); + return 0; +} + +/* + * The caller couldn't find a page that contains the key we're looking + * for. We combine a block's worth of items around the key in all the + * forest btrees and store them in pages. After filtering out deletions + * and duplicates, we insert any resulting pages which don't overlap + * with existing cached pages. + * + * We only insert uncached regions because this is called with cluster + * locks held, but without locking the cache. The regions we read can + * be stale with respect to the current cache, which can be read and + * dirtied by other cluster lock holders on our node, but the cluster + * locks protect the stable items we read. + * + * There's also the exciting case where a reader can populate the cache + * with stale old persistent data which was read before another local + * cluster lock holder was able to read, dirty, write, and then shrink + * the cache. In this case the cache couldn't be cleared by lock + * invalidation because the caller is actively holding the lock. But + * shrinking could evict the cache within the held lock. So we record + * that we're an active reader in the range covered by the lock and + * shrink will refuse to reclaim any pages that intersect with our read. + */ +static int read_pages(struct super_block *sb, struct item_cache_info *cinf, + struct scoutfs_key *key, struct scoutfs_lock *lock) +{ + struct rb_root root = RB_ROOT; + struct active_reader active; + struct cached_page *right = NULL; + struct cached_page *pg; + struct cached_page *rd; + struct cached_item *item; + struct scoutfs_key start; + struct scoutfs_key end; + struct scoutfs_key inf; + struct scoutfs_key edge; + struct rb_node **pnode; + struct rb_node *par; + struct rb_node *pg_tmp; + struct rb_node *item_tmp; + int pgi; + int ret; + + /* stop shrink from freeing new clean data, would let us cache stale */ + active.start = lock->start; + active.end = lock->end; + spin_lock(&cinf->active_lock); + active_rbtree_walk(&cinf->active_root, &active.start, &active.end, + &par, &pnode); + rbtree_insert(&active.node, par, pnode, &cinf->active_root); + spin_unlock(&cinf->active_lock); + + /* start with an empty page that covers the whole lock */ + pg = alloc_pg(sb, 0); + if (!pg) { + ret = -ENOMEM; + goto out; + } + pg->start = lock->start; + pg->end = lock->end; + rbtree_insert(&pg->node, NULL, &root.rb_node, &root); + + ret = scoutfs_forest_read_items(sb, lock, key, &start, &end, + read_page_item, &root); + if (ret < 0) + goto out; + + /* clean up our read items and pages before locking */ + for_each_page_safe(&root, pg, pg_tmp) { + + /* trim any items we read outside the read range */ + scoutfs_key_set_zeros(&inf); + edge = start; + scoutfs_key_dec(&edge); + pgi = trim_page_intersection(sb, cinf, pg, NULL, &inf, &edge); + if (pgi != PGI_INSIDE) { + scoutfs_key_set_ones(&inf); + edge = end; + scoutfs_key_inc(&edge); + pgi = trim_page_intersection(sb, cinf, pg, NULL, &edge, + &inf); + } + if (pgi == PGI_INSIDE) { + rbtree_erase(&pg->node, &root); + put_pg(sb, pg); + continue; + } + + /* drop deletion items, we don't need them in the cache */ + for_each_item_safe(&pg->item_root, item, item_tmp) { + if (item->deletion) + erase_item(pg, item); + } + } + +retry: + write_lock(&cinf->rwlock); + + while ((rd = first_page(&root))) { + + pg = page_rbtree_walk(sb, &cinf->pg_root, &rd->start, &rd->end, + NULL, NULL, &par, &pnode); + if (!pg) { + /* insert read pages that don't intersect */ + rbtree_erase(&rd->node, &root); + rbtree_insert(&rd->node, par, pnode, &cinf->pg_root); + lru_accessed(sb, cinf, rd); + continue; + } + + pgi = trim_page_intersection(sb, cinf, rd, right, &pg->start, + &pg->end); + if (pgi == PGI_INSIDE) { + rbtree_erase(&rd->node, &root); + put_pg(sb, rd); + + } else if (pgi == PGI_BISECT_NEEDED) { + write_unlock(&cinf->rwlock); + right = alloc_pg(sb, 0); + if (!right) { + ret = -ENOMEM; + goto out; + } + goto retry; + + } else if (pgi == PGI_BISECT) { + page_rbtree_walk(sb, &root, &right->start, &right->end, + NULL, NULL, &par, &pnode); + rbtree_insert(&right->node, par, pnode, &root); + right = NULL; + } + } + + write_unlock(&cinf->rwlock); + + ret = 0; +out: + spin_lock(&cinf->active_lock); + rbtree_erase(&active.node, &cinf->active_root); + spin_unlock(&cinf->active_lock); + + /* free any pages we left dangling on error */ + for_each_page_safe(&root, rd, pg_tmp) { + rbtree_erase(&rd->node, &root); + put_pg(sb, rd); + } + + put_pg(sb, right); + + return ret; +} + +/* + * Get a locked cached page for the caller to work with. This populates + * the cache on misses and can ensure that the locked page has enough + * room for an item allocation for the caller. Unfortunately, sparse + * doesn't seem to deal very well with the pattern of conditional lock + * acquisition. Callers manually add __acquire. + */ +static int get_cached_page(struct super_block *sb, + struct item_cache_info *cinf, + struct scoutfs_lock *lock, struct scoutfs_key *key, + bool write, bool alloc, int val_len, + struct cached_page **pg_ret) +{ + struct cached_page *pg = NULL; + struct rb_node **pnode; + struct rb_node *par; + int ret; + + if (WARN_ON_ONCE(alloc && !write)) + return -EINVAL; + + pg = get_pcpu_page(sb, cinf, key, write); + if (pg) { + __acquire(pg->rwlock); + if (!alloc || page_has_room(pg, val_len)) + goto found; + + if (write) + write_unlock(&pg->rwlock); + else + read_unlock(&pg->rwlock); + pg = NULL; + } + +retry: + read_lock(&cinf->rwlock); + + pg = page_rbtree_walk(sb, &cinf->pg_root, key, key, NULL, NULL, + &par, &pnode); + if (pg == NULL) { + read_unlock(&cinf->rwlock); + if (lock->mode == SCOUTFS_LOCK_WRITE_ONLY) + ret = cache_empty_page(sb, cinf, key, &lock->start, + &lock->end); + else + ret = read_pages(sb, cinf, key, lock); + if (ret < 0) + goto out; + goto retry; + } + + if (write) + write_lock(&pg->rwlock); + else + read_lock(&pg->rwlock); + + if (alloc && !page_has_room(pg, val_len)) { + read_unlock(&cinf->rwlock); + if (write) + write_unlock(&pg->rwlock); + else + read_unlock(&pg->rwlock); + + ret = try_split_page(sb, cinf, key, val_len); + if (ret < 0) + goto out; + goto retry; + } + + read_unlock(&cinf->rwlock); + + add_pcpu_page(sb, cinf, pg); +found: + __release(pg_rwlock); + lru_accessed(sb, cinf, pg); + ret = 0; +out: + if (ret < 0) + *pg_ret = NULL; + else + *pg_ret = pg; + return ret; +} + +static int lock_safe(struct scoutfs_lock *lock, struct scoutfs_key *key, + int mode) +{ + if (WARN_ON_ONCE(!scoutfs_lock_protected(lock, key, mode))) + return -EINVAL; + else + return 0; +} + +/* + * Copy the cached item's value into the caller's value. The number of + * bytes copied is returned. A null val returns 0. + */ +static int copy_val(void *dst, int dst_len, void *src, int src_len) +{ + int ret; + + BUG_ON(dst_len < 0 || src_len < 0); + + ret = min(dst_len, src_len); + if (ret) + memcpy(dst, src, ret); + return ret; +} + +/* + * Find an item with the given key and copy its value to the caller. + * The amount of bytes copied is returned which can be 0 or truncated if + * the caller's buffer isn't big enough. + */ +int scoutfs_item_lookup(struct super_block *sb, struct scoutfs_key *key, + void *val, int val_len, struct scoutfs_lock *lock) +{ + DECLARE_ITEM_CACHE_INFO(sb, cinf); + struct cached_item *item; + struct cached_page *pg; + int ret; + + scoutfs_inc_counter(sb, item_lookup); + + if ((ret = lock_safe(lock, key, SCOUTFS_LOCK_READ))) + goto out; + + ret = get_cached_page(sb, cinf, lock, key, false, false, 0, &pg); + if (ret < 0) + goto out; + __acquire(&pg->rwlock); + + item = item_rbtree_walk(&pg->item_root, key, NULL, NULL, NULL); + if (!item || item->deletion) + ret = -ENOENT; + else + ret = copy_val(val, val_len, item->val, item->val_len); + + read_unlock(&pg->rwlock); +out: + return ret; +} + +int scoutfs_item_lookup_exact(struct super_block *sb, struct scoutfs_key *key, + void *val, int val_len, + struct scoutfs_lock *lock) +{ + int ret; + + ret = scoutfs_item_lookup(sb, key, val, val_len, lock); + if (ret == val_len) + ret = 0; + else if (ret >= 0) + ret = -EIO; + + return ret; +} + +/* + * Return the next item starting with the given key and returning the + * last key at most. + * + * The range covered by the lock also limits the last item that can be + * returned. -ENOENT can be returned when there are no next items + * covered by the lock but there are still items before the last key + * outside of the lock. The caller needs to know to reacquire the next + * lock to continue iteration. + * + * -ENOENT is returned if there are no items between the given and last + * keys inside the range covered by the lock. + * + * The next item's key is copied to the caller's key. + * + * The next item's value is copied into the callers value. The number + * of value bytes copied is returned. The copied value can be truncated + * by the caller's value buffer length. + */ +int scoutfs_item_next(struct super_block *sb, struct scoutfs_key *key, + struct scoutfs_key *last, void *val, int val_len, + struct scoutfs_lock *lock) +{ + DECLARE_ITEM_CACHE_INFO(sb, cinf); + struct cached_item *item; + struct cached_item *next; + struct cached_page *pg = NULL; + struct scoutfs_key pos; + int ret; + + scoutfs_inc_counter(sb, item_next); + + /* use the end key as the last key if it's closer */ + if (scoutfs_key_compare(&lock->end, last) < 0) + last = &lock->end; + + if (scoutfs_key_compare(key, last) > 0) { + ret = -ENOENT; + goto out; + } + + if ((ret = lock_safe(lock, key, SCOUTFS_LOCK_READ))) + goto out; + + pos = *key; + + for (;;) { + ret = get_cached_page(sb, cinf, lock, &pos, false, false, 0, + &pg); + if (ret < 0) + goto out; + __acquire(&pg->rwlock); + + item = item_rbtree_walk(&pg->item_root, &pos, &next, + NULL, NULL) ?: next; + while (item && scoutfs_key_compare(&item->key, last) <= 0) { + if (!item->deletion) { + *key = item->key; + ret = copy_val(val, val_len, item->val, + item->val_len); + goto unlock; + } + + item = next_item(item); + } + + if (scoutfs_key_compare(&pg->end, last) >= 0) { + ret = -ENOENT; + goto unlock; + } + + pos = pg->end; + read_unlock(&pg->rwlock); + + scoutfs_key_inc(&pos); + } + +unlock: + read_unlock(&pg->rwlock); +out: + + return ret; +} + +/* + * Mark the item dirty. Dirtying while holding a transaction pins the + * page holding the item and guarantees that the item can be deleted or + * updated (without increasing the value length) during the transaction + * without errors. + */ +int scoutfs_item_dirty(struct super_block *sb, struct scoutfs_key *key, + struct scoutfs_lock *lock) +{ + DECLARE_ITEM_CACHE_INFO(sb, cinf); + struct cached_item *item; + struct cached_page *pg; + int ret; + + scoutfs_inc_counter(sb, item_dirty); + + if ((ret = lock_safe(lock, key, SCOUTFS_LOCK_WRITE))) + goto out; + + ret = scoutfs_forest_set_bloom_bits(sb, lock); + if (ret < 0) + goto out; + + ret = get_cached_page(sb, cinf, lock, key, true, false, 0, &pg); + if (ret < 0) + goto out; + __acquire(pg->rwlock); + + item = item_rbtree_walk(&pg->item_root, key, NULL, NULL, NULL); + if (!item || item->deletion) { + ret = -ENOENT; + } else { + mark_item_dirty(sb, cinf, pg, NULL, item); + item->liv.vers = cpu_to_le64(lock->write_version); + ret = 0; + } + + write_unlock(&pg->rwlock); +out: + return ret; +} + +/* + * Create a new cached item with the given value. -EEXIST is returned + * if the item already exists. Forcing creates the item without knowldge + * of any existing items.. it doesn't read and can't return -EEXIST. + */ +static int item_create(struct super_block *sb, struct scoutfs_key *key, + void *val, int val_len, struct scoutfs_lock *lock, + int mode, bool force) +{ + DECLARE_ITEM_CACHE_INFO(sb, cinf); + struct scoutfs_log_item_value liv = { + .vers = cpu_to_le64(lock->write_version), + }; + struct cached_item *found; + struct cached_item *item; + struct cached_page *pg; + struct rb_node **pnode; + struct rb_node *par; + int ret; + + scoutfs_inc_counter(sb, item_create); + + if ((ret = lock_safe(lock, key, mode))) + goto out; + + ret = scoutfs_forest_set_bloom_bits(sb, lock); + if (ret < 0) + goto out; + + ret = get_cached_page(sb, cinf, lock, key, true, true, val_len, &pg); + if (ret < 0) + goto out; + __acquire(pg->rwlock); + + found = item_rbtree_walk(&pg->item_root, key, NULL, &par, &pnode); + if (!force && found && !found->deletion) { + ret = -EEXIST; + goto unlock; + } + + item = alloc_item(pg, key, &liv, val, val_len); + rbtree_insert(&item->node, par, pnode, &pg->item_root); + mark_item_dirty(sb, cinf, pg, NULL, item); + + if (found) { + item->persistent = found->persistent; + clear_item_dirty(sb, cinf, pg, found); + erase_item(pg, found); + } + + if (force) + item->persistent = 1; + + ret = 0; +unlock: + write_unlock(&pg->rwlock); +out: + return ret; +} + +int scoutfs_item_create(struct super_block *sb, struct scoutfs_key *key, + void *val, int val_len, struct scoutfs_lock *lock) +{ + return item_create(sb, key, val, val_len, lock, + SCOUTFS_LOCK_READ, false); +} + +int scoutfs_item_create_force(struct super_block *sb, struct scoutfs_key *key, + void *val, int val_len, + struct scoutfs_lock *lock) +{ + return item_create(sb, key, val, val_len, lock, + SCOUTFS_LOCK_WRITE_ONLY, true); +} + +/* + * Update an item with a new value. If the new value is smaller and the + * item is dirty then this is guaranteed to succeed. It can fail if the + * item doesn't exist or it gets errors reading or allocating new pages + * for a larger value. + */ +int scoutfs_item_update(struct super_block *sb, struct scoutfs_key *key, + void *val, int val_len, struct scoutfs_lock *lock) +{ + DECLARE_ITEM_CACHE_INFO(sb, cinf); + struct scoutfs_log_item_value liv = { + .vers = cpu_to_le64(lock->write_version), + }; + struct cached_item *item; + struct cached_item *found; + struct cached_page *pg; + struct rb_node **pnode; + struct rb_node *par; + int ret; + + scoutfs_inc_counter(sb, item_update); + + if ((ret = lock_safe(lock, key, SCOUTFS_LOCK_WRITE))) + goto out; + + ret = scoutfs_forest_set_bloom_bits(sb, lock); + if (ret < 0) + goto out; + + ret = get_cached_page(sb, cinf, lock, key, true, true, val_len, &pg); + if (ret < 0) + goto out; + __acquire(pg->rwlock); + + found = item_rbtree_walk(&pg->item_root, key, NULL, &par, &pnode); + if (!found || found->deletion) { + ret = -ENOENT; + goto unlock; + } + + if (val_len <= found->val_len) { + if (val_len) + memcpy(found->val, val, val_len); + if (val_len < found->val_len) + pg->erased_bytes += found->val_len - val_len; + found->val_len = val_len; + found->liv.vers = liv.vers; + mark_item_dirty(sb, cinf, pg, NULL, found); + } else { + item = alloc_item(pg, key, &liv, val, val_len); + item->persistent = found->persistent; + rbtree_insert(&item->node, par, pnode, &pg->item_root); + mark_item_dirty(sb, cinf, pg, NULL, item); + + clear_item_dirty(sb, cinf, pg, found); + erase_item(pg, found); + } + + ret = 0; +unlock: + write_unlock(&pg->rwlock); +out: + return ret; +} + +/* + * Delete an item from the cache. We can leave behind a dirty deletion + * item if there is a persistent item that needs to be overwritten. + * This can't fail if the caller knows that the item exists and it has + * been dirtied during the transaction it holds. If we're forcing then + * we're not reading the old state of the item and have to create a + * deletion item if there isn't one already cached. + */ +static int item_delete(struct super_block *sb, struct scoutfs_key *key, + struct scoutfs_lock *lock, int mode, bool force) +{ + DECLARE_ITEM_CACHE_INFO(sb, cinf); + struct scoutfs_log_item_value liv = { + .vers = cpu_to_le64(lock->write_version), + }; + struct cached_item *item; + struct cached_page *pg; + struct rb_node **pnode; + struct rb_node *par; + int ret; + + scoutfs_inc_counter(sb, item_delete); + + if ((ret = lock_safe(lock, key, mode))) + goto out; + + ret = scoutfs_forest_set_bloom_bits(sb, lock); + if (ret < 0) + goto out; + + ret = get_cached_page(sb, cinf, lock, key, true, force, 0, &pg); + if (ret < 0) + goto out; + __acquire(pg->rwlock); + + item = item_rbtree_walk(&pg->item_root, key, NULL, &par, &pnode); + if (!force && (!item || item->deletion)) { + ret = -ENOENT; + goto unlock; + } + + if (!item) { + item = alloc_item(pg, key, &liv, NULL, 0); + rbtree_insert(&item->node, par, pnode, &pg->item_root); + } + + if (force) + item->persistent = 1; + + if (!item->persistent) { + /* can just forget items that aren't yet persistent */ + clear_item_dirty(sb, cinf, pg, item); + erase_item(pg, item); + } else { + /* must emit deletion to clobber old persistent item */ + item->liv.vers = cpu_to_le64(lock->write_version); + item->liv.flags |= SCOUTFS_LOG_ITEM_FLAG_DELETION; + item->deletion = 1; + pg->erased_bytes += item->val_len; + item->val_len = 0; + mark_item_dirty(sb, cinf, pg, NULL, item); + } + + ret = 0; +unlock: + write_unlock(&pg->rwlock); +out: + return ret; +} + +int scoutfs_item_delete(struct super_block *sb, struct scoutfs_key *key, + struct scoutfs_lock *lock) +{ + return item_delete(sb, key, lock, SCOUTFS_LOCK_WRITE, false); +} + +int scoutfs_item_delete_force(struct super_block *sb, struct scoutfs_key *key, + struct scoutfs_lock *lock) +{ + return item_delete(sb, key, lock, SCOUTFS_LOCK_WRITE_ONLY, true); +} + +u64 scoutfs_item_dirty_pages(struct super_block *sb) +{ + DECLARE_ITEM_CACHE_INFO(sb, cinf); + + return (u64)atomic_read(&cinf->dirty_pages); +} + +static int cmp_pg_start(void *priv, struct list_head *A, struct list_head *B) +{ + struct cached_page *a = list_entry(A, struct cached_page, dirty_head); + struct cached_page *b = list_entry(B, struct cached_page, dirty_head); + + return scoutfs_key_compare(&a->start, &b->start); +} + +static int cmp_item_key(void *priv, struct list_head *A, struct list_head *B) +{ + struct cached_item *a = list_entry(A, struct cached_item, dirty_head); + struct cached_item *b = list_entry(B, struct cached_item, dirty_head); + + return scoutfs_key_compare(&a->key, &b->key); +} + +/* + * Write all the dirty items into dirty blocks in the forest of btrees. + * If this succeeds then the dirty blocks can be submitted to commit + * their transaction. If this returns an error then the dirty blocks + * could have a partial set of the dirty items and result in an + * inconsistent state. The blocks should only be committed once all the + * dirty items have been written. + * + * This is called during transaction commit which prevents item writers + * from entering a transaction and dirtying items. The set of dirty + * items will be constant. + * + * But the pages that contain the dirty items can be changing. A + * neighbouring read lock can be invalidated and require bisecting a + * page, moving dirty items to a new page. That new page will be put + * after the original page on the dirty list. This will be done under + * the page rwlock and the global dirty_lock. + * + * We first sort the pages by their keys, then lock each page and copy + * its items into a private allocated singly-linked list of the items to + * dirty. Once we have that we can hand it off to the forest of btrees + * to write into items without causing any contention with other page + * users. + */ +int scoutfs_item_write_dirty(struct super_block *sb) +{ + DECLARE_ITEM_CACHE_INFO(sb, cinf); + struct scoutfs_btree_item_list *first; + struct scoutfs_btree_item_list **prev; + struct scoutfs_btree_item_list *lst; + struct cached_item *item; + struct cached_page *pg; + struct page *second = NULL; + struct page *page; + LIST_HEAD(pages); + LIST_HEAD(pos); + u64 max_vers = 0; + int val_len; + int bytes; + int off; + int ret; + + /* we're relying on struct layout to prepend item value headers */ + BUILD_BUG_ON(offsetof(struct cached_item, val) != + (offsetof(struct cached_item, liv) + + member_sizeof(struct cached_item, liv))); + + if (atomic_read(&cinf->dirty_pages) == 0) + return 0; + + scoutfs_inc_counter(sb, item_write_dirty); + + /* sort page dirty list by keys */ + read_lock(&cinf->rwlock); + spin_lock(&cinf->dirty_lock); + + /* sort cached pages by key, add our pos head */ + list_sort(NULL, &cinf->dirty_list, cmp_pg_start); + list_add(&pos, &cinf->dirty_list); + + read_unlock(&cinf->rwlock); + spin_unlock(&cinf->dirty_lock); + + page = alloc_page(GFP_NOFS); + if (!page) { + ret = -ENOMEM; + goto out; + } + list_add(&page->list, &pages); + + first = NULL; + prev = &first; + off = 0; + + while (!list_empty_careful(&pos)) { + if (!second) { + second = alloc_page(GFP_NOFS); + if (!second) { + ret = -ENOMEM; + goto out; + } + list_add(&second->list, &pages); + } + + /* read lock next sorted page, we're only dirty_list user */ + + spin_lock(&cinf->dirty_lock); + pg = list_entry(pos.next, struct cached_page, dirty_head); + if (!read_trylock(&pg->rwlock)) { + spin_unlock(&cinf->dirty_lock); + cpu_relax(); + continue; + } + spin_unlock(&cinf->dirty_lock); + + list_sort(NULL, &pg->dirty_list, cmp_item_key); + + list_for_each_entry(item, &pg->dirty_list, dirty_head) { + val_len = sizeof(item->liv) + item->val_len; + bytes = offsetof(struct scoutfs_btree_item_list, + val[val_len]); + max_vers = max(max_vers, le64_to_cpu(item->liv.vers)); + + if (off + bytes > PAGE_SIZE) { + page = second; + second = NULL; + off = 0; + } + + lst = (void *)page_address(page) + off; + off += round_up(bytes, CACHED_ITEM_ALIGN); + + lst->next = NULL; + *prev = lst; + prev = &lst->next; + + lst->key = item->key; + lst->val_len = val_len; + memcpy(lst->val, &item->liv, val_len); + } + + spin_lock(&cinf->dirty_lock); + if (pg->dirty_head.next == &cinf->dirty_list) + list_del_init(&pos); + else + list_move(&pos, &pg->dirty_head); + spin_unlock(&cinf->dirty_lock); + + read_unlock(&pg->rwlock); + } + + /* store max item vers in forest's log_trees */ + scoutfs_forest_set_max_vers(sb, max_vers); + + /* write all the dirty items into log btree blocks */ + ret = scoutfs_forest_insert_list(sb, first); +out: + list_for_each_entry_safe(page, second, &pages, list) { + list_del_init(&page->list); + __free_page(page); + } + + return ret; +} + +/* + * The caller has successfully committed all the dirty btree blocks that + * contained the currently dirty items. Clear all the dirty items and + * pages. + */ +int scoutfs_item_write_done(struct super_block *sb) +{ + DECLARE_ITEM_CACHE_INFO(sb, cinf); + struct cached_item *item; + struct cached_item *tmp; + struct cached_page *pg; + +retry: + spin_lock(&cinf->dirty_lock); + + while ((pg = list_first_entry_or_null(&cinf->dirty_list, + struct cached_page, + dirty_head))) { + + if (!write_trylock(&pg->rwlock)) { + spin_unlock(&cinf->dirty_lock); + cpu_relax(); + goto retry; + } + + spin_unlock(&cinf->dirty_lock); + + list_for_each_entry_safe(item, tmp, &pg->dirty_list, + dirty_head) { + clear_item_dirty(sb, cinf, pg, item); + + /* free deletion items */ + if (item->deletion) + erase_item(pg, item); + else + item->persistent = 1; + } + + write_unlock(&pg->rwlock); + + spin_lock(&cinf->dirty_lock); + } + + spin_unlock(&cinf->dirty_lock); + + return 0; +} + +/* + * Return true if the item cache covers the given range and set *dirty + * to true if any items in the cached range are dirty. + * + * This is relatively rarely called as locks are granted to make sure + * that we *don't* have existing cache covered by the lock which then + * must be inconsistent. Finding pages is the critical error case, + * under correct operation this will be a read locked walk of the page + * rbtree that doesn't find anything. + */ +bool scoutfs_item_range_cached(struct super_block *sb, + struct scoutfs_key *start, + struct scoutfs_key *end, bool *dirty) +{ + DECLARE_ITEM_CACHE_INFO(sb, cinf); + struct cached_item *item; + struct cached_page *pg; + struct scoutfs_key pos; + bool cached; + + cached = false; + *dirty = false; + pos = *start; + + read_lock(&cinf->rwlock); + + while (!(*dirty) && scoutfs_key_compare(&pos, end) <= 0 && + (pg = page_rbtree_walk(sb, &cinf->pg_root, &pos, end, NULL, NULL, + NULL, NULL))) { + cached = true; + + read_lock(&pg->rwlock); + read_unlock(&cinf->rwlock); + + /* the dirty list isn't sorted :/ */ + list_for_each_entry(item, &pg->dirty_list, dirty_head) { + if (!scoutfs_key_compare_ranges(&item->key, &item->key, + start, end)) { + *dirty = true; + break; + } + } + + pos = pg->end; + scoutfs_key_inc(&pos); + + read_unlock(&pg->rwlock); + read_lock(&cinf->rwlock); + } + + read_unlock(&cinf->rwlock); + + return cached; +} + +/* + * Remove the cached items in the given range. We drop pages that are + * fully inside the range and trim any pages that intersect it. This is + * being by locking for a lock that can't be used so there can't be item + * calls within the range. It can race with all our other page uses. + */ +void scoutfs_item_invalidate(struct super_block *sb, struct scoutfs_key *start, + struct scoutfs_key *end) +{ + DECLARE_ITEM_CACHE_INFO(sb, cinf); + struct cached_page *right = NULL; + struct cached_page *pg; + struct rb_node **pnode; + struct rb_node *par; + int pgi; + + scoutfs_inc_counter(sb, item_invalidate); + +retry: + write_lock(&cinf->rwlock); + + while ((pg = page_rbtree_walk(sb, &cinf->pg_root, start, end, NULL, + NULL, &par, &pnode))) { + + scoutfs_inc_counter(sb, item_invalidate_page); + + write_lock(&pg->rwlock); + + pgi = trim_page_intersection(sb, cinf, pg, right, start, end); + BUG_ON(pgi == PGI_DISJOINT); /* walk wouldn't ret disjoint */ + + if (pgi == PGI_INSIDE) { + /* free entirely invalidated page */ + lru_remove(sb, cinf, pg); + rbtree_erase(&pg->node, &cinf->pg_root); + invalidate_pcpu_page(pg); + write_unlock(&pg->rwlock); + put_pg(sb, pg); + continue; + + } else if (pgi == PGI_BISECT_NEEDED) { + /* allocate so we can bisect a larger page */ + write_unlock(&cinf->rwlock); + write_unlock(&pg->rwlock); + right = alloc_pg(sb, __GFP_NOFAIL); + goto retry; + + } else if (pgi == PGI_BISECT) { + /* inv was entirely inside page, done after bisect */ + write_trylock_will_succeed(&right->rwlock); + rbtree_insert(&right->node, par, pnode, &cinf->pg_root); + write_unlock(&right->rwlock); + write_unlock(&pg->rwlock); + lru_accessed(sb, cinf, right); + right = NULL; + break; + } + + /* OLAP trimmed edge, keep searching */ + write_unlock(&pg->rwlock); + } + + write_unlock(&cinf->rwlock); + + put_pg(sb, right); +} + +/* + * Shrink the size the item cache. We're operating against the fast + * path lock ordering and we skip pages if we can't acquire locks. + * Similarly, we can run into dirty pages or pages which intersect with + * active readers that we can't shrink and also choose to skip. + */ +static int item_lru_shrink(struct shrinker *shrink, + struct shrink_control *sc) +{ + struct item_cache_info *cinf = container_of(shrink, + struct item_cache_info, + shrinker); + struct super_block *sb = cinf->sb; + struct active_reader *active; + struct cached_page *tmp; + struct cached_page *pg; + LIST_HEAD(list); + int nr; + + if (sc->nr_to_scan == 0) + goto out; + nr = sc->nr_to_scan; + + write_lock(&cinf->rwlock); + spin_lock(&cinf->lru_lock); + + list_for_each_entry_safe(pg, tmp, &cinf->lru_list, lru_head) { + + /* can't invalidate ranges being read, reader might be stale */ + spin_lock(&cinf->active_lock); + active = active_rbtree_walk(&cinf->active_root, &pg->start, + &pg->end, NULL, NULL); + spin_unlock(&cinf->active_lock); + if (active) { + scoutfs_inc_counter(sb, item_shrink_page_reader); + continue; + } + + if (!write_trylock(&pg->rwlock)) { + scoutfs_inc_counter(sb, item_shrink_page_trylock); + continue; + } + + if (!list_empty(&pg->dirty_list)) { + scoutfs_inc_counter(sb, item_shrink_page_dirty); + write_unlock(&pg->rwlock); + continue; + } + + scoutfs_inc_counter(sb, item_shrink_page); + + __lru_remove(sb, cinf, pg); + rbtree_erase(&pg->node, &cinf->pg_root); + list_move_tail(&pg->lru_head, &list); + invalidate_pcpu_page(pg); + write_unlock(&pg->rwlock); + + if (--nr == 0) + break; + } + + write_unlock(&cinf->rwlock); + spin_unlock(&cinf->lru_lock); + + list_for_each_entry_safe(pg, tmp, &list, lru_head) { + list_del_init(&pg->lru_head); + put_pg(sb, pg); + } +out: + return min_t(unsigned long, cinf->lru_pages, INT_MAX); +} + +static int item_cpu_callback(struct notifier_block *nfb, + unsigned long action, void *hcpu) +{ + struct item_cache_info *cinf = container_of(nfb, + struct item_cache_info, + notifier); + struct super_block *sb = cinf->sb; + unsigned long cpu = (unsigned long)hcpu; + + if (action == CPU_DEAD) + drop_pcpu_pages(sb, cinf, cpu); + + return NOTIFY_OK; +} + +int scoutfs_item_setup(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct item_cache_info *cinf; + int cpu; + + cinf = kzalloc(sizeof(struct item_cache_info), GFP_KERNEL); + if (!cinf) + return -ENOMEM; + + cinf->sb = sb; + rwlock_init(&cinf->rwlock); + cinf->pg_root = RB_ROOT; + spin_lock_init(&cinf->dirty_lock); + INIT_LIST_HEAD(&cinf->dirty_list); + atomic_set(&cinf->dirty_pages, 0); + spin_lock_init(&cinf->lru_lock); + INIT_LIST_HEAD(&cinf->lru_list); + spin_lock_init(&cinf->active_lock); + cinf->active_root = RB_ROOT; + + cinf->pcpu_pages = alloc_percpu(struct item_percpu_pages); + if (!cinf->pcpu_pages) + return -ENOMEM; + + for_each_possible_cpu(cpu) + init_pcpu_pages(cinf, cpu); + + cinf->shrinker.shrink = item_lru_shrink; + cinf->shrinker.seeks = DEFAULT_SEEKS; + register_shrinker(&cinf->shrinker); + cinf->notifier.notifier_call = item_cpu_callback; + register_hotcpu_notifier(&cinf->notifier); + + sbi->item_cache_info = cinf; + return 0; +} + +/* + * There must be no more item callers at this point. + */ +void scoutfs_item_destroy(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct item_cache_info *cinf = sbi->item_cache_info; + struct cached_page *tmp; + struct cached_page *pg; + int cpu; + + if (cinf) { + BUG_ON(!RB_EMPTY_ROOT(&cinf->active_root)); + + unregister_hotcpu_notifier(&cinf->notifier); + unregister_shrinker(&cinf->shrinker); + + for_each_possible_cpu(cpu) + drop_pcpu_pages(sb, cinf, cpu); + free_percpu(cinf->pcpu_pages); + + rbtree_postorder_for_each_entry_safe(pg, tmp, &cinf->pg_root, + node) { + RB_CLEAR_NODE(&pg->node); + INIT_LIST_HEAD(&pg->lru_head); + INIT_LIST_HEAD(&pg->dirty_list); + INIT_LIST_HEAD(&pg->dirty_head); + put_pg(sb, pg); + } + + kfree(cinf); + sbi->item_cache_info = NULL; + } +} diff --git a/kmod/src/item.h b/kmod/src/item.h new file mode 100644 index 00000000..ae4046e7 --- /dev/null +++ b/kmod/src/item.h @@ -0,0 +1,39 @@ +#ifndef _SCOUTFS_ITEM_H_ +#define _SCOUTFS_ITEM_H_ + +int scoutfs_item_lookup(struct super_block *sb, struct scoutfs_key *key, + void *val, int val_len, struct scoutfs_lock *lock); +int scoutfs_item_lookup_exact(struct super_block *sb, struct scoutfs_key *key, + void *val, int val_len, + struct scoutfs_lock *lock); +int scoutfs_item_next(struct super_block *sb, struct scoutfs_key *key, + struct scoutfs_key *last, void *val, int val_len, + struct scoutfs_lock *lock); +int scoutfs_item_dirty(struct super_block *sb, struct scoutfs_key *key, + struct scoutfs_lock *lock); +int scoutfs_item_create(struct super_block *sb, struct scoutfs_key *key, + void *val, int val_len, struct scoutfs_lock *lock); +int scoutfs_item_create_force(struct super_block *sb, struct scoutfs_key *key, + void *val, int val_len, + struct scoutfs_lock *lock); +int scoutfs_item_update(struct super_block *sb, struct scoutfs_key *key, + void *val, int val_len, struct scoutfs_lock *lock); +int scoutfs_item_delete(struct super_block *sb, struct scoutfs_key *key, + struct scoutfs_lock *lock); +int scoutfs_item_delete_force(struct super_block *sb, + struct scoutfs_key *key, + struct scoutfs_lock *lock); + +u64 scoutfs_item_dirty_pages(struct super_block *sb); +int scoutfs_item_write_dirty(struct super_block *sb); +int scoutfs_item_write_done(struct super_block *sb); +bool scoutfs_item_range_cached(struct super_block *sb, + struct scoutfs_key *start, + struct scoutfs_key *end, bool *dirty); +void scoutfs_item_invalidate(struct super_block *sb, struct scoutfs_key *start, + struct scoutfs_key *end); + +int scoutfs_item_setup(struct super_block *sb); +void scoutfs_item_destroy(struct super_block *sb); + +#endif diff --git a/kmod/src/kernelcompat.h b/kmod/src/kernelcompat.h new file mode 100644 index 00000000..6899f684 --- /dev/null +++ b/kmod/src/kernelcompat.h @@ -0,0 +1,49 @@ +#ifndef _SCOUTFS_KERNELCOMPAT_H_ +#define _SCOUTFS_KERNELCOMPAT_H_ + +#ifndef KC_ITERATE_DIR_CONTEXT +#include +typedef filldir_t kc_readdir_ctx_t; +#define KC_DECLARE_READDIR(name, file, dirent, ctx) name(file, dirent, ctx) +#define KC_FOP_READDIR readdir +#define kc_readdir_pos(filp, ctx) (filp)->f_pos +#define kc_dir_emit_dots(file, dirent, ctx) dir_emit_dots(file, dirent, ctx) +#define kc_dir_emit(ctx, dirent, name, name_len, pos, ino, dt) \ + (ctx(dirent, name, name_len, pos, ino, dt) == 0) +#else +typedef struct dir_context * kc_readdir_ctx_t; +#define KC_DECLARE_READDIR(name, file, dirent, ctx) name(file, ctx) +#define KC_FOP_READDIR iterate +#define kc_readdir_pos(filp, ctx) (ctx)->pos +#define kc_dir_emit_dots(file, dirent, ctx) dir_emit_dots(file, ctx) +#define kc_dir_emit(ctx, dirent, name, name_len, pos, ino, dt) \ + dir_emit(ctx, name, name_len, ino, dt) +#endif + +#ifndef KC_DIR_EMIT_DOTS +/* + * Kernels before ->iterate and don't have dir_emit_dots so we give them + * one that works with the ->readdir() filldir() method. + */ +static inline int dir_emit_dots(struct file *file, void *dirent, + filldir_t filldir) +{ + if (file->f_pos == 0) { + if (filldir(dirent, ".", 1, 1, + file->f_path.dentry->d_inode->i_ino, DT_DIR)) + return 0; + file->f_pos = 1; + } + + if (file->f_pos == 1) { + if (filldir(dirent, "..", 2, 1, + parent_ino(file->f_path.dentry), DT_DIR)) + return 0; + file->f_pos = 2; + } + + return 1; +} +#endif + +#endif diff --git a/kmod/src/key.h b/kmod/src/key.h new file mode 100644 index 00000000..5ea4dd4c --- /dev/null +++ b/kmod/src/key.h @@ -0,0 +1,206 @@ +#ifndef _SCOUTFS_KEY_H_ +#define _SCOUTFS_KEY_H_ + +#include +#include "format.h" +#include "cmp.h" +#include "endian_swap.h" + +#define SK_FMT "%u.%llu.%u.%llu.%llu.%u" + +/* This does not support null keys */ +#define SK_ARG(key) (key)->sk_zone, \ + le64_to_cpu((key)->_sk_first), \ + (key)->sk_type, \ + le64_to_cpu((key)->_sk_second), \ + le64_to_cpu((key)->_sk_third), \ + (key)->_sk_fourth + +/* userspace trace event printing doesn't like arguments with structure + * field references. So we explode structures into their fields instead + * of + */ +#define sk_trace_define(name) \ + __field(__u8, name##_zone) \ + __field(__u64, name##_first) \ + __field(__u8, name##_type) \ + __field(__u64, name##_second) \ + __field(__u64, name##_third) \ + __field(__u8, name##_fourth) + +#define sk_trace_assign(name, key) \ +do { \ + __typeof__(key) _key = (key); \ + if (_key) { \ + __entry->name##_zone = _key->sk_zone; \ + __entry->name##_first = le64_to_cpu(_key->_sk_first); \ + __entry->name##_type = _key->sk_type; \ + __entry->name##_second = le64_to_cpu(_key->_sk_second);\ + __entry->name##_third = le64_to_cpu(_key->_sk_third); \ + __entry->name##_fourth = _key->_sk_fourth; \ + } else { \ + __entry->name##_zone = 0; \ + __entry->name##_first = 0; \ + __entry->name##_type = 0; \ + __entry->name##_second = 0; \ + __entry->name##_third = 0; \ + __entry->name##_fourth = 0; \ + } \ +} while (0) + +#define sk_trace_args(name) \ + __entry->name##_zone, __entry->name##_first, __entry->name##_type, \ + __entry->name##_second, __entry->name##_third, __entry->name##_fourth + +/* + * copy fields between keys with the same fields but different types. + * The destination type might have internal padding so we zero it. + */ +#define scoutfs_key_copy_types(a, b) \ +do { \ + __typeof__(a) _to = (a); \ + __typeof__(b) _from = (b); \ + \ + memset(_to, 0, sizeof(*_to)); \ + _to->sk_zone = _from->sk_zone; \ + _to->_sk_first = _from->_sk_first; \ + _to->sk_type = _from->sk_type; \ + _to->_sk_second = _from->_sk_second; \ + _to->_sk_third = _from->_sk_third; \ + _to->_sk_fourth = _from->_sk_fourth; \ +} while (0) + +static inline void scoutfs_key_set_zeros(struct scoutfs_key *key) +{ + key->sk_zone = 0; + key->_sk_first = 0; + key->sk_type = 0; + key->_sk_second = 0; + key->_sk_third = 0; + key->_sk_fourth = 0; + memset(key->__pad, 0, sizeof(key->__pad)); +} + +static inline bool scoutfs_key_is_zeros(struct scoutfs_key *key) +{ + return key->sk_zone == 0 && key->_sk_first == 0 && key->sk_type == 0 && + key->_sk_second == 0 && key->_sk_third == 0 && + key->_sk_fourth == 0; +} + +static inline void scoutfs_key_copy_or_zeros(struct scoutfs_key *dst, + struct scoutfs_key *src) +{ + if (src) + *dst = *src; + else + scoutfs_key_set_zeros(dst); +} + +static inline void scoutfs_key_set_ones(struct scoutfs_key *key) +{ + key->sk_zone = U8_MAX; + key->_sk_first = cpu_to_le64(U64_MAX); + key->sk_type = U8_MAX; + key->_sk_second = cpu_to_le64(U64_MAX); + key->_sk_third = cpu_to_le64(U64_MAX); + key->_sk_fourth = U8_MAX; + memset(key->__pad, 0, sizeof(key->__pad)); +} + +/* + * Return a -1/0/1 comparison of keys. + * + * It turns out that these ternary chains are consistently cheaper than + * other alternatives across keys that first differ in any of the + * values. Say maybe 20% faster than memcmp. + */ +static inline int scoutfs_key_compare(struct scoutfs_key *a, + struct scoutfs_key *b) +{ + return scoutfs_cmp(a->sk_zone, b->sk_zone) ?: + scoutfs_cmp(le64_to_cpu(a->_sk_first), le64_to_cpu(b->_sk_first)) ?: + scoutfs_cmp(a->sk_type, b->sk_type) ?: + scoutfs_cmp(le64_to_cpu(a->_sk_second), le64_to_cpu(b->_sk_second)) ?: + scoutfs_cmp(le64_to_cpu(a->_sk_third), le64_to_cpu(b->_sk_third)) ?: + scoutfs_cmp(a->_sk_fourth, b->_sk_fourth); +} + +/* + * Compare ranges of keys where overlapping is equality. Returns: + * -1: a_end < b_start + * 1: a_start > b_end + * else 0: ranges overlap + */ +static inline int scoutfs_key_compare_ranges(struct scoutfs_key *a_start, + struct scoutfs_key *a_end, + struct scoutfs_key *b_start, + struct scoutfs_key *b_end) +{ + return scoutfs_key_compare(a_end, b_start) < 0 ? -1 : + scoutfs_key_compare(a_start, b_end) > 0 ? 1 : + 0; +} + +static inline void scoutfs_key_inc(struct scoutfs_key *key) +{ + if (++key->_sk_fourth != 0) + return; + + le64_add_cpu(&key->_sk_third, 1); + if (key->_sk_third != 0) + return; + + le64_add_cpu(&key->_sk_second, 1); + if (key->_sk_second != 0) + return; + + if (++key->sk_type != 0) + return; + + le64_add_cpu(&key->_sk_first, 1); + if (key->_sk_first != 0) + return; + + key->sk_zone++; +} + +static inline void scoutfs_key_dec(struct scoutfs_key *key) +{ + if (--key->_sk_fourth != U8_MAX) + return; + + le64_add_cpu(&key->_sk_third, -1); + if (key->_sk_third != cpu_to_le64(U64_MAX)) + return; + + le64_add_cpu(&key->_sk_second, -1); + if (key->_sk_second != cpu_to_le64(U64_MAX)) + return; + + if (--key->sk_type != U8_MAX) + return; + + le64_add_cpu(&key->_sk_first, -1); + if (key->_sk_first != cpu_to_le64(U64_MAX)) + return; + + key->sk_zone--; +} + +/* + * Some key types are used by multiple subsystems and shouldn't have + * duplicate private key init functions. + */ + +static inline void scoutfs_key_init_log_trees(struct scoutfs_key *key, + u64 rid, u64 nr) +{ + *key = (struct scoutfs_key) { + .sk_zone = SCOUTFS_LOG_TREES_ZONE, + .sklt_rid = cpu_to_le64(rid), + .sklt_nr = cpu_to_le64(nr), + }; +} + +#endif diff --git a/kmod/src/lock.c b/kmod/src/lock.c new file mode 100644 index 00000000..309a1152 --- /dev/null +++ b/kmod/src/lock.c @@ -0,0 +1,1709 @@ +/* + * Copyright (C) 2019 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include /* a rhel shed.h needed preempt_offset? */ +#include +#include +#include +#include +#include + +#include "super.h" +#include "lock.h" +#include "scoutfs_trace.h" +#include "msg.h" +#include "cmp.h" +#include "inode.h" +#include "trans.h" +#include "counters.h" +#include "endian_swap.h" +#include "triggers.h" +#include "tseq.h" +#include "client.h" +#include "data.h" +#include "xattr.h" +#include "item.h" + +/* + * scoutfs uses a lock service to manage item cache consistency between + * nodes. We map ranges of item keys to locks and use each lock's modes + * to govern what can be done with the items under the lock. Locks are + * held by mounts who populate, write out, and invalidate their caches + * as they acquire and release locks. + * + * The locking client in a mount sends lock requests to the server. The + * server eventually responds with a response that grants access to the + * lock. The server then sends a revoke request to the client which + * tells it the mode that it should reduce the lock to. If it removes + * all access to the lock (by revoking it down to a null mode) then the + * lock is freed. + * + * Memory pressure on the client can cause the client to request a null + * mode from the server so that once its granted the lock can be freed. + * + * So far we've only needed a minimal trylock. We return -EAGAIN if a + * lock attempt can't immediately match an existing granted lock. This + * is fine for the only rare user which can back out of its lock + * inversion and retry with a full blocking lock. + * + * Lock recovery is initiated by the server when it recognizes that + * we're reconnecting to it while a previous server left a persistenr + * record of us. We resend all our pending requests which are deferred + * until recovery finishes. The server sends us a recovery request and + * we respond with all our locks. Our resent requests are processed + * relative to that lock state we resend. + */ + +#define GRACE_PERIOD_KT ms_to_ktime(2) + +/* + * allocated per-super, freed on unmount. + */ +struct lock_info { + struct super_block *sb; + spinlock_t lock; + bool shutdown; + struct rb_root lock_tree; + struct rb_root lock_range_tree; + struct shrinker shrinker; + struct list_head lru_list; + unsigned long long lru_nr; + struct workqueue_struct *workq; + struct work_struct grant_work; + struct list_head grant_list; + struct delayed_work inv_dwork; + struct list_head inv_list; + struct work_struct shrink_work; + struct list_head shrink_list; + atomic64_t next_refresh_gen; + struct dentry *tseq_dentry; + struct scoutfs_tseq_tree tseq_tree; +}; + +#define DECLARE_LOCK_INFO(sb, name) \ + struct lock_info *name = SCOUTFS_SB(sb)->lock_info + +static bool lock_mode_invalid(enum scoutfs_lock_mode mode) +{ + return (unsigned)mode >= SCOUTFS_LOCK_INVALID; +} + +static bool lock_mode_can_read(enum scoutfs_lock_mode mode) +{ + return mode == SCOUTFS_LOCK_READ || mode == SCOUTFS_LOCK_WRITE; +} + +static bool lock_mode_can_write(enum scoutfs_lock_mode mode) +{ + return mode == SCOUTFS_LOCK_WRITE || mode == SCOUTFS_LOCK_WRITE_ONLY; +} + +/* + * Returns true if a lock with the granted mode can satisfy a requested + * mode. This is directional. A read lock is satisfied by a write lock + * but not vice versa. + */ +static bool lock_modes_match(int granted, int requested) +{ + return (granted == requested) || + (granted == SCOUTFS_LOCK_WRITE && + requested == SCOUTFS_LOCK_READ); +} + +/* + * invalidate cached data associated with an inode whose lock is going + * away. + */ +static void invalidate_inode(struct super_block *sb, u64 ino) +{ + struct inode *inode; + + inode = scoutfs_ilookup(sb, ino); + if (inode) { + scoutfs_inc_counter(sb, lock_invalidate_inode); + if (S_ISREG(inode->i_mode)) { + truncate_inode_pages(inode->i_mapping, 0); + scoutfs_data_wait_changed(inode); + } + iput(inode); + } +} + +/* + * Invalidate caches associated with this lock. Either we're + * invalidating a write to a read or we're invalidating to null. We + * always have to write out dirty items if there are any. We can only + * leave cached items behind in the case of invalidating to a read lock. + */ +static int lock_invalidate(struct super_block *sb, struct scoutfs_lock *lock, + enum scoutfs_lock_mode prev, enum scoutfs_lock_mode mode) +{ + struct scoutfs_lock_coverage *cov; + struct scoutfs_lock_coverage *tmp; + u64 ino, last; + int ret = 0; + + trace_scoutfs_lock_invalidate(sb, lock); + + /* verify assertion made by comment above */ + BUG_ON(!(prev == SCOUTFS_LOCK_WRITE && mode == SCOUTFS_LOCK_READ) && + mode != SCOUTFS_LOCK_NULL); + + /* sync when a write lock could have dirtied the current transaction */ + if (lock_mode_can_write(prev) && + (lock->dirty_trans_seq == scoutfs_trans_sample_seq(sb))) { + scoutfs_inc_counter(sb, lock_invalidate_sync); + ret = scoutfs_trans_sync(sb, 1); + if (ret < 0) + return ret; + } + + /* have to invalidate if we're not in the only usable case */ + if (!(prev == SCOUTFS_LOCK_WRITE && mode == SCOUTFS_LOCK_READ)) { +retry: + /* remove cov items to tell users that their cache is stale */ + spin_lock(&lock->cov_list_lock); + list_for_each_entry_safe(cov, tmp, &lock->cov_list, head) { + if (!spin_trylock(&cov->cov_lock)) { + spin_unlock(&lock->cov_list_lock); + cpu_relax(); + goto retry; + } + list_del_init(&cov->head); + cov->lock = NULL; + spin_unlock(&cov->cov_lock); + scoutfs_inc_counter(sb, lock_invalidate_coverage); + } + spin_unlock(&lock->cov_list_lock); + + if (lock->start.sk_zone == SCOUTFS_FS_ZONE) { + ino = le64_to_cpu(lock->start.ski_ino); + last = le64_to_cpu(lock->end.ski_ino); + while (ino <= last) { + invalidate_inode(sb, ino); + ino++; + } + } + + scoutfs_item_invalidate(sb, &lock->start, &lock->end); + } + + return ret; +} + +static void lock_free(struct lock_info *linfo, struct scoutfs_lock *lock) +{ + struct super_block *sb = lock->sb; + + assert_spin_locked(&linfo->lock); + + trace_scoutfs_lock_free(sb, lock); + scoutfs_inc_counter(sb, lock_free); + + /* manually checking lock_idle gives identifying line numbers */ + BUG_ON(lock->request_pending); + BUG_ON(lock->invalidate_pending); + BUG_ON(lock->waiters[SCOUTFS_LOCK_READ]); + BUG_ON(lock->waiters[SCOUTFS_LOCK_WRITE]); + BUG_ON(lock->waiters[SCOUTFS_LOCK_WRITE_ONLY]); + BUG_ON(lock->users[SCOUTFS_LOCK_READ]); + BUG_ON(lock->users[SCOUTFS_LOCK_WRITE]); + BUG_ON(lock->users[SCOUTFS_LOCK_WRITE_ONLY]); + BUG_ON(!linfo->shutdown && lock->mode != SCOUTFS_LOCK_NULL); + BUG_ON(!RB_EMPTY_NODE(&lock->node)); + BUG_ON(!RB_EMPTY_NODE(&lock->range_node)); + BUG_ON(!list_empty(&lock->lru_head)); + BUG_ON(!list_empty(&lock->grant_head)); + BUG_ON(!list_empty(&lock->inv_head)); + BUG_ON(!list_empty(&lock->shrink_head)); + BUG_ON(!list_empty(&lock->cov_list)); + + kfree(lock); +} + +static struct scoutfs_lock *lock_alloc(struct super_block *sb, + struct scoutfs_key *start, + struct scoutfs_key *end) + +{ + struct scoutfs_lock *lock; + + if (WARN_ON_ONCE(!start || !end)) + return NULL; + + lock = kzalloc(sizeof(struct scoutfs_lock), GFP_NOFS); + if (lock == NULL) + return NULL; + + scoutfs_inc_counter(sb, lock_alloc); + + RB_CLEAR_NODE(&lock->node); + RB_CLEAR_NODE(&lock->range_node); + INIT_LIST_HEAD(&lock->lru_head); + INIT_LIST_HEAD(&lock->grant_head); + INIT_LIST_HEAD(&lock->inv_head); + INIT_LIST_HEAD(&lock->shrink_head); + spin_lock_init(&lock->cov_list_lock); + INIT_LIST_HEAD(&lock->cov_list); + + lock->start = *start; + lock->end = *end; + lock->sb = sb; + init_waitqueue_head(&lock->waitq); + lock->mode = SCOUTFS_LOCK_NULL; + + atomic64_set(&lock->forest_bloom_nr, 0); + + trace_scoutfs_lock_alloc(sb, lock); + + return lock; +} + +static void lock_inc_count(unsigned int *counts, enum scoutfs_lock_mode mode) +{ + BUG_ON(mode < 0 || mode >= SCOUTFS_LOCK_NR_MODES); + counts[mode]++; +} + +static void lock_dec_count(unsigned int *counts, enum scoutfs_lock_mode mode) +{ + BUG_ON(mode < 0 || mode >= SCOUTFS_LOCK_NR_MODES); + counts[mode]--; +} + +/* + * Returns true if all the actively used modes are satisfied by a lock + * of the given granted mode. + */ +static bool lock_counts_match(int granted, unsigned int *counts) +{ + enum scoutfs_lock_mode mode; + + for (mode = 0; mode < SCOUTFS_LOCK_NR_MODES; mode++) { + if (counts[mode] && !lock_modes_match(granted, mode)) + return false; + } + + return true; +} + +/* + * Returns true if there are any mode counts that match with the desired + * mode. There can be other non-matching counts as well but we're only + * testing for the existence of any matching counts. + */ +static bool lock_count_match_exists(int desired, unsigned int *counts) +{ + enum scoutfs_lock_mode mode; + + for (mode = 0; mode < SCOUTFS_LOCK_NR_MODES; mode++) { + if (counts[mode] && lock_modes_match(desired, mode)) + return true; + } + + return false; +} + +/* + * An idle lock has nothing going on. It can be present in the lru and + * can be freed by the final put when it has a null mode. + */ +static bool lock_idle(struct scoutfs_lock *lock) +{ + enum scoutfs_lock_mode mode; + + if (lock->request_pending || lock->invalidate_pending) + return false; + + for (mode = 0; mode < SCOUTFS_LOCK_NR_MODES; mode++) { + if (lock->waiters[mode] || lock->users[mode]) + return false; + } + + return true; +} + +static bool insert_range_node(struct super_block *sb, struct scoutfs_lock *ins) +{ + DECLARE_LOCK_INFO(sb, linfo); + struct rb_root *root = &linfo->lock_range_tree; + struct rb_node **node = &root->rb_node; + struct rb_node *parent = NULL; + struct scoutfs_lock *lock; + int cmp; + + while (*node) { + parent = *node; + lock = container_of(*node, struct scoutfs_lock, range_node); + + cmp = scoutfs_key_compare_ranges(&ins->start, &ins->end, + &lock->start, &lock->end); + if (WARN_ON_ONCE(cmp == 0)) { + scoutfs_warn(sb, "inserting lock start "SK_FMT" end "SK_FMT" overlaps with existing lock start "SK_FMT" end "SK_FMT, + SK_ARG(&ins->start), SK_ARG(&ins->end), + SK_ARG(&lock->start), SK_ARG(&lock->end)); + return false; + } + + if (cmp < 0) + node = &(*node)->rb_left; + else + node = &(*node)->rb_right; + } + + + rb_link_node(&ins->range_node, parent, node); + rb_insert_color(&ins->range_node, root); + + return true; +} + +/* returns true if the lock was inserted at its start key */ +static bool lock_insert(struct super_block *sb, struct scoutfs_lock *ins) +{ + DECLARE_LOCK_INFO(sb, linfo); + struct scoutfs_lock *lock; + struct rb_node *parent; + struct rb_node **node; + int cmp; + + assert_spin_locked(&linfo->lock); + + node = &linfo->lock_tree.rb_node; + parent = NULL; + while (*node) { + parent = *node; + lock = container_of(*node, struct scoutfs_lock, node); + + cmp = scoutfs_key_compare(&ins->start, &lock->start); + if (cmp < 0) + node = &(*node)->rb_left; + else if (cmp > 0) + node = &(*node)->rb_right; + else + return false; + } + + if (!insert_range_node(sb, ins)) + return false; + + rb_link_node(&ins->node, parent, node); + rb_insert_color(&ins->node, &linfo->lock_tree); + + scoutfs_tseq_add(&linfo->tseq_tree, &ins->tseq_entry); + + return true; +} + +static void lock_remove(struct lock_info *linfo, struct scoutfs_lock *lock) +{ + assert_spin_locked(&linfo->lock); + + rb_erase(&lock->node, &linfo->lock_tree); + RB_CLEAR_NODE(&lock->node); + rb_erase(&lock->range_node, &linfo->lock_range_tree); + RB_CLEAR_NODE(&lock->range_node); + + scoutfs_tseq_del(&linfo->tseq_tree, &lock->tseq_entry); +} + +static struct scoutfs_lock *lock_lookup(struct super_block *sb, + struct scoutfs_key *start, + struct scoutfs_lock **next) +{ + DECLARE_LOCK_INFO(sb, linfo); + struct rb_node *node = linfo->lock_tree.rb_node; + struct scoutfs_lock *lock; + int cmp; + + assert_spin_locked(&linfo->lock); + + if (next) + *next = NULL; + + while (node) { + lock = container_of(node, struct scoutfs_lock, node); + + cmp = scoutfs_key_compare(start, &lock->start); + if (cmp < 0) { + if (next) + *next = lock; + node = node->rb_left; + } else if (cmp > 0) { + node = node->rb_right; + } else { + return lock; + } + } + + return NULL; +} + +static void __lock_del_lru(struct lock_info *linfo, struct scoutfs_lock *lock) +{ + assert_spin_locked(&linfo->lock); + + if (!list_empty(&lock->lru_head)) { + list_del_init(&lock->lru_head); + linfo->lru_nr--; + } +} + +/* + * Get a lock and remove it from the lru. The caller must set state on + * the lock that indicates that it's busy before dropping the lock. + * Then later they call add_lru_or_free once they've cleared that state. + */ +static struct scoutfs_lock *get_lock(struct super_block *sb, + struct scoutfs_key *start) +{ + DECLARE_LOCK_INFO(sb, linfo); + struct scoutfs_lock *lock; + + assert_spin_locked(&linfo->lock); + + lock = lock_lookup(sb, start, NULL); + if (lock) + __lock_del_lru(linfo, lock); + + return lock; +} + +/* + * Get a lock, creating it if it doesn't exist. The caller must treat + * the lock like it came from get lock (mark sate, drop lock, clear + * state, put lock). Allocated locks aren't on the lru. + */ +static struct scoutfs_lock *create_lock(struct super_block *sb, + struct scoutfs_key *start, + struct scoutfs_key *end) +{ + DECLARE_LOCK_INFO(sb, linfo); + struct scoutfs_lock *lock; + + assert_spin_locked(&linfo->lock); + + lock = get_lock(sb, start); + if (!lock) { + spin_unlock(&linfo->lock); + lock = lock_alloc(sb, start, end); + spin_lock(&linfo->lock); + + if (lock) { + if (!lock_insert(sb, lock)) { + lock_free(linfo, lock); + lock = get_lock(sb, start); + } + } + } + + return lock; +} + +/* + * The caller is done using a lock and has cleared state that used to + * indicate that the lock wasn't idle. If it really is idle then we + * either free it if it's null or put it back on the lru. + */ +static void put_lock(struct lock_info *linfo,struct scoutfs_lock *lock) +{ + assert_spin_locked(&linfo->lock); + + if (lock_idle(lock)) { + if (lock->mode != SCOUTFS_LOCK_NULL) { + list_add_tail(&lock->lru_head, &linfo->lru_list); + linfo->lru_nr++; + } else { + lock_remove(linfo, lock); + lock_free(linfo, lock); + } + } +} + +/* + * Locks have a grace period that extends after activity and prevents + * invalidation. It's intended to let nodes do reasonable batches of + * work as locks ping pong between nodes that are doing conflicting + * work. + */ +static void extend_grace(struct super_block *sb, struct scoutfs_lock *lock) +{ + ktime_t now = ktime_get(); + + if (ktime_after(now, lock->grace_deadline)) + scoutfs_inc_counter(sb, lock_grace_set); + else + scoutfs_inc_counter(sb, lock_grace_extended); + + lock->grace_deadline = ktime_add(now, GRACE_PERIOD_KT); +} + +static void queue_grant_work(struct lock_info *linfo) +{ + assert_spin_locked(&linfo->lock); + + if (!list_empty(&linfo->grant_list) && !linfo->shutdown) + queue_work(linfo->workq, &linfo->grant_work); +} + +/* + * We immediately queue work on the assumption that the caller might + * have made a change (set a lock mode) which can let one of the + * invalidating locks make forward progress, even if other locks are + * waiting for their grace period to elapse. It's a trade-off between + * invalidation latency and burning cpu repeatedly finding that locks + * are still in their grace period. + */ +static void queue_inv_work(struct lock_info *linfo) +{ + assert_spin_locked(&linfo->lock); + + if (!list_empty(&linfo->inv_list) && !linfo->shutdown) + mod_delayed_work(linfo->workq, &linfo->inv_dwork, 0); +} + +/* + * The given lock is processing a received a grant response. Trigger a + * bug if the cache is inconsistent. + * + * We only have two modes that can create dirty items. We can't have + * dirty items when transitioning from write_only to write because the + * writer can't trust the cached items in the cache for reading. And we + * don't currently transition directly from write to write_only, we + * first go through null. So if we have dirty items as we're granted a + * mode it's always incorrect. + * + * And we can't have cached items that we're going to use for reading if + * the previous mode didn't allow reading. + * + * Inconsistencies have come from all sorts of bugs: invalidation missed + * items, the cache was populated outside of locking coverage, lock + * holders performed the wrong item operations under their lock, + * overlapping locks, out of order granting or invalidating, etc. + */ +static void bug_on_inconsistent_grant_cache(struct super_block *sb, + struct scoutfs_lock *lock, + int old_mode, int new_mode) +{ + bool cached; + bool dirty; + + cached = scoutfs_item_range_cached(sb, &lock->start, &lock->end, + &dirty); + if (dirty || + (cached && (!lock_mode_can_read(old_mode) || + !lock_mode_can_read(new_mode)))) { + scoutfs_err(sb, "granted lock item cache inconsistency, cached %u dirty %u old_mode %d new_mode %d: start "SK_FMT" end "SK_FMT" refresh_gen %llu mode %u waiters: rd %u wr %u wo %u users: rd %u wr %u wo %u", + cached, dirty, old_mode, new_mode, SK_ARG(&lock->start), + SK_ARG(&lock->end), lock->refresh_gen, lock->mode, + lock->waiters[SCOUTFS_LOCK_READ], + lock->waiters[SCOUTFS_LOCK_WRITE], + lock->waiters[SCOUTFS_LOCK_WRITE_ONLY], + lock->users[SCOUTFS_LOCK_READ], + lock->users[SCOUTFS_LOCK_WRITE], + lock->users[SCOUTFS_LOCK_WRITE_ONLY]); + BUG(); + } +} + +/* + * Each lock has received a grant response message from the server. + * + * Grant responses can be reordered with incoming invalidation requests + * from the server so we have to be careful to only set the new mode + * once the old mode matches. + * + * We extend the grace period as we grant the lock if there is a waiting + * locker who can use the lock. This stops invalidation from pulling + * the granted lock out from under the requester, resulting in a lot of + * churn with no forward progress. Using the grace period avoids having + * to identify a specific waiter and give it an acquired lock. It's + * also very similar to waking up the locker and having it win the race + * against the invalidation. In that case they'd extend the grace + * period anyway as they unlock. + */ +static void lock_grant_worker(struct work_struct *work) +{ + struct lock_info *linfo = container_of(work, struct lock_info, + grant_work); + struct super_block *sb = linfo->sb; + struct scoutfs_net_lock_grant_response *gr; + struct scoutfs_net_lock *nl; + struct scoutfs_lock *lock; + struct scoutfs_lock *tmp; + + scoutfs_inc_counter(sb, lock_grant_work); + + spin_lock(&linfo->lock); + + list_for_each_entry_safe(lock, tmp, &linfo->grant_list, grant_head) { + gr = &lock->grant_resp; + nl = &lock->grant_resp.nl; + + /* wait for reordered invalidation to finish */ + if (lock->mode != nl->old_mode) + continue; + + bug_on_inconsistent_grant_cache(sb, lock, nl->old_mode, + nl->new_mode); + + if (!lock_mode_can_read(nl->old_mode) && + lock_mode_can_read(nl->new_mode)) { + lock->refresh_gen = + atomic64_inc_return(&linfo->next_refresh_gen); + } + + lock->request_pending = 0; + lock->mode = nl->new_mode; + lock->write_version = le64_to_cpu(nl->write_version); + lock->roots = gr->roots; + + if (lock_count_match_exists(nl->new_mode, lock->waiters)) + extend_grace(sb, lock); + + trace_scoutfs_lock_granted(sb, lock); + list_del_init(&lock->grant_head); + wake_up(&lock->waitq); + put_lock(linfo, lock); + } + + /* invalidations might be waiting for our reordered grant */ + queue_inv_work(linfo); + spin_unlock(&linfo->lock); +} + +/* + * The client is receiving a grant response message from the server. We + * find the lock, record the response, and add it to the list for grant + * work to process. + */ +int scoutfs_lock_grant_response(struct super_block *sb, + struct scoutfs_net_lock_grant_response *gr) +{ + struct scoutfs_net_lock *nl = &gr->nl; + DECLARE_LOCK_INFO(sb, linfo); + struct scoutfs_lock *lock; + + scoutfs_inc_counter(sb, lock_grant_response); + + spin_lock(&linfo->lock); + + /* lock must already be busy with request_pending */ + lock = lock_lookup(sb, &nl->key, NULL); + BUG_ON(!lock); + trace_scoutfs_lock_grant_response(sb, lock); + BUG_ON(!lock->request_pending); + + lock->grant_resp = *gr; + list_add_tail(&lock->grant_head, &linfo->grant_list); + queue_grant_work(linfo); + + spin_unlock(&linfo->lock); + + return 0; +} + +/* + * Each lock has received a lock invalidation request from the server + * which specifies a new mode for the lock. The server will only send + * one invalidation request at a time for each lock. + * + * This is an unsolicited request from the server so it can arrive at + * any time after we make the server aware of the lock by initially + * requesting it. We wait for users of the current mode to unlock + * before invalidating. + * + * This can arrive on behalf of our request for a mode that conflicts + * with our current mode. We have to proceed while we have a request + * pending. We can also be racing with shrink requests being sent while + * we're invalidating. + * + * This can be processed concurrently and experience reordering with a + * grant response sent back-to-back from the server. We carefully only + * invalidate once the lock mode matches what the server told us to + * invalidate. + * + * We delay invalidation processing until a grace period has elapsed + * since the last unlock. The intent is to let users do a reasonable + * batch of work before dropping the lock. Continuous unlocking can + * continuously extend the deadline. + * + * Before we start invalidating the lock we set the lock to the new + * mode, preventing further incompatible users of the old mode from + * using the lock while we're invalidating. + * + * This does a lot of serialized inode invalidation in one context and + * performs a lot of repeated calls to sync. It would be nice to get + * some concurrent inode invalidation and to more carefully only call + * sync when needed. + */ +static void lock_invalidate_worker(struct work_struct *work) +{ + struct lock_info *linfo = container_of(work, struct lock_info, + inv_dwork.work); + struct super_block *sb = linfo->sb; + struct scoutfs_net_lock *nl; + struct scoutfs_lock *lock; + struct scoutfs_lock *tmp; + unsigned long delay = MAX_JIFFY_OFFSET; + ktime_t now = ktime_get(); + ktime_t deadline; + LIST_HEAD(ready); + u64 net_id; + int ret; + + scoutfs_inc_counter(sb, lock_invalidate_work); + + spin_lock(&linfo->lock); + + list_for_each_entry_safe(lock, tmp, &linfo->inv_list, inv_head) { + nl = &lock->inv_nl; + + /* skip if grace hasn't elapsed, record earliest */ + deadline = lock->grace_deadline; + if (ktime_before(now, deadline)) { + delay = min(delay, + nsecs_to_jiffies(ktime_to_ns( + ktime_sub(deadline, now)))); + scoutfs_inc_counter(linfo->sb, lock_grace_wait); + continue; + } + + /* wait for reordered grant to finish */ + if (lock->mode != nl->old_mode) + continue; + + /* wait until incompatible holders unlock */ + if (!lock_counts_match(nl->new_mode, lock->users)) + continue; + + /* set the new mode, no incompatible users during inval */ + lock->mode = nl->new_mode; + + /* move everyone that's ready to our private list */ + list_move_tail(&lock->inv_head, &ready); + } + + spin_unlock(&linfo->lock); + + if (list_empty(&ready)) + goto out; + + /* invalidate once the lock is read */ + list_for_each_entry(lock, &ready, inv_head) { + nl = &lock->inv_nl; + net_id = lock->inv_net_id; + + ret = lock_invalidate(sb, lock, nl->old_mode, nl->new_mode); + BUG_ON(ret); + + /* respond with the key and modes from the request */ + ret = scoutfs_client_lock_response(sb, net_id, nl); + BUG_ON(ret); + + scoutfs_inc_counter(sb, lock_invalidate_response); + } + + /* and finish all the invalidated locks */ + spin_lock(&linfo->lock); + + list_for_each_entry_safe(lock, tmp, &ready, inv_head) { + list_del_init(&lock->inv_head); + + lock->invalidate_pending = 0; + trace_scoutfs_lock_invalidated(sb, lock); + wake_up(&lock->waitq); + put_lock(linfo, lock); + } + + /* grant might have been waiting for invalidate request */ + queue_grant_work(linfo); + spin_unlock(&linfo->lock); + +out: + /* queue delayed work if invalidations waiting on grace deadline */ + if (delay != MAX_JIFFY_OFFSET) + queue_delayed_work(linfo->workq, &linfo->inv_dwork, delay); +} + +/* + * Record an incoming invalidate request from the server and add its lock + * to the list for processing. + * + * This is trusting the server and will crash if it's sent bad requests :/ + */ +int scoutfs_lock_invalidate_request(struct super_block *sb, u64 net_id, + struct scoutfs_net_lock *nl) +{ + DECLARE_LOCK_INFO(sb, linfo); + struct scoutfs_lock *lock; + + scoutfs_inc_counter(sb, lock_invalidate_request); + + spin_lock(&linfo->lock); + lock = get_lock(sb, &nl->key); + BUG_ON(!lock); + if (lock) { + BUG_ON(lock->invalidate_pending); + lock->invalidate_pending = 1; + lock->inv_nl = *nl; + lock->inv_net_id = net_id; + list_add_tail(&lock->inv_head, &linfo->inv_list); + trace_scoutfs_lock_invalidate_request(sb, lock); + queue_inv_work(linfo); + } + spin_unlock(&linfo->lock); + + return 0; +} + +/* + * The server is asking us to send them as many locks as we can starting + * with the given key. We'll send a response with 0 locks to indicate + * that we've sent all our locks. This is called in client processing + * so the client won't try to reconnect to another server until we + * return. + */ +int scoutfs_lock_recover_request(struct super_block *sb, u64 net_id, + struct scoutfs_key *key) +{ + DECLARE_LOCK_INFO(sb, linfo); + struct scoutfs_net_lock_recover *nlr; + struct scoutfs_lock *lock; + struct scoutfs_lock *next; + struct rb_node *node; + int ret; + int i; + + scoutfs_inc_counter(sb, lock_recover_request); + + nlr = kmalloc(offsetof(struct scoutfs_net_lock_recover, + locks[SCOUTFS_NET_LOCK_MAX_RECOVER_NR]), + GFP_NOFS); + if (!nlr) + return -ENOMEM; + + spin_lock(&linfo->lock); + + lock = lock_lookup(sb, key, &next) ?: next; + + for (i = 0; lock && i < SCOUTFS_NET_LOCK_MAX_RECOVER_NR; i++) { + + nlr->locks[i].key = lock->start; + nlr->locks[i].write_version = cpu_to_le64(lock->write_version); + nlr->locks[i].old_mode = lock->mode; + nlr->locks[i].new_mode = lock->mode; + + node = rb_next(&lock->node); + if (node) + lock = rb_entry(node, struct scoutfs_lock, node); + else + lock = NULL; + } + + nlr->nr = cpu_to_le16(i); + + spin_unlock(&linfo->lock); + + ret = scoutfs_client_lock_recover_response(sb, net_id, nlr); + kfree(nlr); + return ret; +} + +static bool lock_wait_cond(struct super_block *sb, struct scoutfs_lock *lock, + enum scoutfs_lock_mode mode) +{ + DECLARE_LOCK_INFO(sb, linfo); + bool wake; + + spin_lock(&linfo->lock); + wake = linfo->shutdown || lock_modes_match(lock->mode, mode) || + !lock->request_pending; + spin_unlock(&linfo->lock); + + if (!wake) + scoutfs_inc_counter(sb, lock_wait); + + return wake; +} + +static bool lock_flags_invalid(int flags) +{ + return flags & SCOUTFS_LKF_INVALID; +} + +/* + * Acquire a coherent lock on the given range of keys. On success the + * caller can use the given mode to interact with the item cache. While + * holding the lock the cache won't be invalidated and other conflicting + * lock users will be serialized. The item cache can be invalidated + * once the lock is unlocked. + * + * If we don't have a granted lock then we send a request for our + * desired mode if there isn't one in flight already. This can be + * racing with an invalidation request from the server. The server + * won't process our request until it receives our invalidation + * response. + */ +static int lock_key_range(struct super_block *sb, enum scoutfs_lock_mode mode, int flags, + struct scoutfs_key *start, struct scoutfs_key *end, + struct scoutfs_lock **ret_lock) +{ + DECLARE_LOCK_INFO(sb, linfo); + struct scoutfs_lock *lock; + struct scoutfs_net_lock nl; + bool should_send; + int ret; + + scoutfs_inc_counter(sb, lock_lock); + + *ret_lock = NULL; + + if (WARN_ON_ONCE(!start || !end) || + WARN_ON_ONCE(lock_mode_invalid(mode)) || + WARN_ON_ONCE(lock_flags_invalid(flags))) + return -EINVAL; + + /* maybe catch _setup() and _shutdown order mistakes */ + if (WARN_ON_ONCE(!linfo || linfo->shutdown)) + return -ENOLCK; + + /* have to lock before entering transactions */ + if (WARN_ON_ONCE(scoutfs_trans_held())) + return -EDEADLK; + + spin_lock(&linfo->lock); + + /* drops and re-acquires lock if it allocates */ + lock = create_lock(sb, start, end); + if (!lock) { + ret = -ENOMEM; + goto out_unlock; + } + + /* the waiters count is only used by debugging output */ + lock_inc_count(lock->waiters, mode); + + for (;;) { + if (linfo->shutdown) { + ret = -ESHUTDOWN; + break; + } + + /* the fast path where we can use the granted mode */ + if (lock_modes_match(lock->mode, mode)) { + lock_inc_count(lock->users, mode); + *ret_lock = lock; + ret = 0; + break; + } + + /* non-blocking callers don't wait or send requests */ + if (flags & SCOUTFS_LKF_NONBLOCK) { + scoutfs_inc_counter(sb, lock_nonblock_eagain); + ret = -EAGAIN; + break; + } + + if (!lock->request_pending) { + lock->request_pending = 1; + should_send = true; + } else { + should_send = false; + } + + spin_unlock(&linfo->lock); + + if (should_send) { + nl.key = lock->start; + nl.old_mode = lock->mode; + nl.new_mode = mode; + + ret = scoutfs_client_lock_request(sb, &nl); + if (ret) { + spin_lock(&linfo->lock); + lock->request_pending = 0; + break; + } + scoutfs_inc_counter(sb, lock_grant_request); + } + + trace_scoutfs_lock_wait(sb, lock); + + ret = wait_event_interruptible(lock->waitq, + lock_wait_cond(sb, lock, mode)); + spin_lock(&linfo->lock); + if (ret) + break; + } + + lock_dec_count(lock->waiters, mode); + + if (ret == 0) + trace_scoutfs_lock_locked(sb, lock); + wake_up(&lock->waitq); + put_lock(linfo, lock); + +out_unlock: + spin_unlock(&linfo->lock); + + if (ret && ret != -EAGAIN && ret != -ERESTARTSYS) + scoutfs_inc_counter(sb, lock_lock_error); + + return ret; +} + +int scoutfs_lock_ino(struct super_block *sb, enum scoutfs_lock_mode mode, int flags, u64 ino, + struct scoutfs_lock **ret_lock) +{ + struct scoutfs_key start; + struct scoutfs_key end; + + scoutfs_key_set_zeros(&start); + start.sk_zone = SCOUTFS_FS_ZONE; + start.ski_ino = cpu_to_le64(ino & ~(u64)SCOUTFS_LOCK_INODE_GROUP_MASK); + + scoutfs_key_set_ones(&end); + end.sk_zone = SCOUTFS_FS_ZONE; + end.ski_ino = cpu_to_le64(ino | SCOUTFS_LOCK_INODE_GROUP_MASK); + + return lock_key_range(sb, mode, flags, &start, &end, ret_lock); +} + +/* + * Acquire a lock on an inode. + * + * _REFRESH_INODE indicates that the caller needs to have the vfs inode + * fields current with respect to lock coverage. The lock's refresh_gen + * is incremented as new locks are acquired and then indicates that an + * old inode with a smaller refresh_gen needs to be refreshed. + */ +int scoutfs_lock_inode(struct super_block *sb, enum scoutfs_lock_mode mode, int flags, + struct inode *inode, struct scoutfs_lock **lock) +{ + int ret; + + ret = scoutfs_lock_ino(sb, mode, flags, scoutfs_ino(inode), lock); + if (ret < 0) + goto out; + + if (flags & SCOUTFS_LKF_REFRESH_INODE) { + ret = scoutfs_inode_refresh(inode, *lock, flags); + if (ret < 0) { + scoutfs_unlock(sb, *lock, mode); + *lock = NULL; + } + } + +out: + return ret; +} + +struct lock_inodes_arg { + struct inode *inode; + struct scoutfs_lock **lockp; +}; + +/* + * All args with inodes go to the front of the array and are then sorted + * by their inode number. + */ +static int cmp_arg(const void *A, const void *B) +{ + const struct lock_inodes_arg *a = A; + const struct lock_inodes_arg *b = B; + + if (a->inode && b->inode) + return scoutfs_cmp_u64s(scoutfs_ino(a->inode), + scoutfs_ino(b->inode)); + + return a->inode ? -1 : b->inode ? 1 : 0; +} + +static void swap_arg(void *A, void *B, int size) +{ + struct lock_inodes_arg *a = A; + struct lock_inodes_arg *b = B; + + swap(*a, *b); +} + +/* + * Lock all the inodes in inode number order. The inode arguments can + * be in any order and can be duplicated or null. This relies on core + * lock matching to efficiently handle duplicate lock attempts of the + * same group. Callers can try to use the lock range keys for all the + * locks they attempt to acquire without knowing that they map to the + * same groups. + * + * On error no locks are held and all pointers are set to null. Lock + * pointers for null inodes are always set to null. + * + * (pretty great collision with d_lock() here) + */ +int scoutfs_lock_inodes(struct super_block *sb, enum scoutfs_lock_mode mode, int flags, + struct inode *a, struct scoutfs_lock **a_lock, + struct inode *b, struct scoutfs_lock **b_lock, + struct inode *c, struct scoutfs_lock **c_lock, + struct inode *d, struct scoutfs_lock **D_lock) +{ + struct lock_inodes_arg args[] = { + {a, a_lock}, {b, b_lock}, {c, c_lock}, {d, D_lock}, + }; + int ret; + int i; + + /* set all lock pointers to null and validating input */ + ret = 0; + for (i = 0; i < ARRAY_SIZE(args); i++) { + if (WARN_ON_ONCE(args[i].inode && !args[i].lockp)) + ret = -EINVAL; + if (args[i].lockp) + *args[i].lockp = NULL; + } + if (ret) + return ret; + + /* sort by having an inode then inode number */ + sort(args, ARRAY_SIZE(args), sizeof(args[0]), cmp_arg, swap_arg); + + /* lock unique inodes */ + for (i = 0; i < ARRAY_SIZE(args) && args[i].inode; i++) { + ret = scoutfs_lock_inode(sb, mode, flags, args[i].inode, + args[i].lockp); + if (ret) + break; + } + + /* unlock on error */ + for (i = ARRAY_SIZE(args) - 1; ret < 0 && i >= 0; i--) { + if (args[i].lockp && *args[i].lockp) { + scoutfs_unlock(sb, *args[i].lockp, mode); + *args[i].lockp = NULL; + } + } + + return ret; +} + +/* + * The rename lock is magical because it's global. + */ +int scoutfs_lock_rename(struct super_block *sb, enum scoutfs_lock_mode mode, int flags, + struct scoutfs_lock **lock) +{ + struct scoutfs_key key = { + .sk_zone = SCOUTFS_LOCK_ZONE, + .sk_type = SCOUTFS_RENAME_TYPE, + }; + + return lock_key_range(sb, mode, flags, &key, &key, lock); +} + +/* + * Set the caller's keys to the range of index item keys that are + * covered by the lock which covers the given index item. + * + * We're trying to strike a balance between minimizing lock + * communication by locking a large number of items and minimizing + * contention and hold times by locking a small number of items. + * + * The seq indexes have natural batching and limits on the number of + * keys per major value. + * + * This can also be used to find items that are covered by the same lock + * because their starting keys are the same. + */ +void scoutfs_lock_get_index_item_range(u8 type, u64 major, u64 ino, + struct scoutfs_key *start, + struct scoutfs_key *end) +{ + u64 start_major = major & ~SCOUTFS_LOCK_SEQ_GROUP_MASK; + u64 end_major = major | SCOUTFS_LOCK_SEQ_GROUP_MASK; + + BUG_ON(type != SCOUTFS_INODE_INDEX_META_SEQ_TYPE && + type != SCOUTFS_INODE_INDEX_DATA_SEQ_TYPE); + + if (start) + scoutfs_inode_init_index_key(start, type, start_major, 0, 0); + + if (end) + scoutfs_inode_init_index_key(end, type, end_major, U32_MAX, + U64_MAX); +} + +/* + * Lock the given index item. We use the index masks to calculate the + * start and end key values that are covered by the lock. + */ +int scoutfs_lock_inode_index(struct super_block *sb, enum scoutfs_lock_mode mode, + u8 type, u64 major, u64 ino, + struct scoutfs_lock **ret_lock) +{ + struct scoutfs_key start; + struct scoutfs_key end; + + scoutfs_lock_get_index_item_range(type, major, ino, &start, &end); + + return lock_key_range(sb, mode, 0, &start, &end, ret_lock); +} + +/* + * The rid lock protects a mount's private persistent items in the rid + * zone. It's held for the duration of the mount. It lets the mount + * modify the rid items at will and signals to other mounts that we're + * still alive and our rid items shouldn't be reclaimed. + * + * Being held for the entire mount prevents other nodes from reclaiming + * our items, like free blocks, when it would make sense for them to be + * able to. Maybe we have a bunch free and they're trying to allocate + * and are getting ENOSPC. + */ +int scoutfs_lock_rid(struct super_block *sb, enum scoutfs_lock_mode mode, int flags, + u64 rid, struct scoutfs_lock **lock) +{ + struct scoutfs_key start; + struct scoutfs_key end; + + scoutfs_key_set_zeros(&start); + start.sk_zone = SCOUTFS_RID_ZONE; + start.sko_rid = cpu_to_le64(rid); + + scoutfs_key_set_ones(&end); + end.sk_zone = SCOUTFS_RID_ZONE; + end.sko_rid = cpu_to_le64(rid); + + return lock_key_range(sb, mode, flags, &start, &end, lock); +} + +/* + * As we unlock we always extend the grace period to give the caller + * another pass at the lock before its invalidated. + */ +void scoutfs_unlock(struct super_block *sb, struct scoutfs_lock *lock, enum scoutfs_lock_mode mode) +{ + DECLARE_LOCK_INFO(sb, linfo); + + if (IS_ERR_OR_NULL(lock)) + return; + + scoutfs_inc_counter(sb, lock_unlock); + + spin_lock(&linfo->lock); + + lock_dec_count(lock->users, mode); + extend_grace(sb, lock); + if (lock_mode_can_write(mode)) + lock->dirty_trans_seq = scoutfs_trans_sample_seq(sb); + + trace_scoutfs_lock_unlock(sb, lock); + wake_up(&lock->waitq); + queue_inv_work(linfo); + put_lock(linfo, lock); + + spin_unlock(&linfo->lock); +} + +void scoutfs_lock_init_coverage(struct scoutfs_lock_coverage *cov) +{ + spin_lock_init(&cov->cov_lock); + cov->lock = NULL; + INIT_LIST_HEAD(&cov->head); +} + +/* + * Record that the given coverage struct is protected by the given lock. + * Once the lock is dropped the coverage list head will be removed and + * callers can use that to see that the cov isn't covered any more. The + * cov might be on another lock so we're careful to remove it. + */ +void scoutfs_lock_add_coverage(struct super_block *sb, + struct scoutfs_lock *lock, + struct scoutfs_lock_coverage *cov) +{ + spin_lock(&cov->cov_lock); + + if (cov->lock) { + spin_lock(&cov->lock->cov_list_lock); + list_del_init(&cov->head); + spin_unlock(&cov->lock->cov_list_lock); + cov->lock = NULL; + } + + cov->lock = lock; + spin_lock(&cov->lock->cov_list_lock); + list_add(&cov->head, &lock->cov_list); + spin_unlock(&cov->lock->cov_list_lock); + + spin_unlock(&cov->cov_lock); +} + +bool scoutfs_lock_is_covered(struct super_block *sb, + struct scoutfs_lock_coverage *cov) +{ + bool covered; + + spin_lock(&cov->cov_lock); + covered = !list_empty_careful(&cov->head); + spin_unlock(&cov->cov_lock); + + return covered; +} + +void scoutfs_lock_del_coverage(struct super_block *sb, + struct scoutfs_lock_coverage *cov) +{ + spin_lock(&cov->cov_lock); + if (cov->lock) { + spin_lock(&cov->lock->cov_list_lock); + list_del_init(&cov->head); + spin_unlock(&cov->lock->cov_list_lock); + cov->lock = NULL; + } + spin_unlock(&cov->cov_lock); +} + +/* + * Returns true if the given lock protects the given access of the given + * key. The lock must have a current granted mode that is compatible + * with the access mode and the access key must be in the lock's key + * range. + * + * This is called by lock holders who's use of the lock must be preventing + * the mode and keys from changing. + */ +bool scoutfs_lock_protected(struct scoutfs_lock *lock, struct scoutfs_key *key, + enum scoutfs_lock_mode mode) +{ + signed char lock_mode = ACCESS_ONCE(lock->mode); + + return lock_modes_match(lock_mode, mode) && + scoutfs_key_compare_ranges(key, key, + &lock->start, &lock->end) == 0; +} + +/* + * The shrink callback got the lock, marked it request_pending, and put + * it on the shrink list. We send a null request and the lock will be + * freed by the response once all users drain. If this races with + * invalidation then the server will only send the grant response once + * the invalidation is finished. + */ +static void lock_shrink_worker(struct work_struct *work) +{ + struct lock_info *linfo = container_of(work, struct lock_info, + shrink_work); + struct super_block *sb = linfo->sb; + struct scoutfs_net_lock nl; + struct scoutfs_lock *lock; + struct scoutfs_lock *tmp; + LIST_HEAD(list); + int ret; + + scoutfs_inc_counter(sb, lock_shrink_work); + + spin_lock(&linfo->lock); + list_splice_init(&linfo->shrink_list, &list); + spin_unlock(&linfo->lock); + + list_for_each_entry_safe(lock, tmp, &list, shrink_head) { + list_del_init(&lock->shrink_head); + + /* unlocked lock access, but should be stable since we queued */ + nl.key = lock->start; + nl.old_mode = lock->mode; + nl.new_mode = SCOUTFS_LOCK_NULL; + + ret = scoutfs_client_lock_request(sb, &nl); + if (ret) { + /* oh well, not freeing */ + scoutfs_inc_counter(sb, lock_shrink_aborted); + + spin_lock(&linfo->lock); + + lock->request_pending = 0; + wake_up(&lock->waitq); + put_lock(linfo, lock); + + spin_unlock(&linfo->lock); + } + } +} + +/* + * Start the shrinking process for locks on the lru. If a lock is on + * the lru then it can't have any active users. We don't want to block + * or allocate here so all we do is get the lock, mark it request + * pending, and kick off the work. The work sends a null request and + * eventually the lock is freed by its response. + * + * Only a racing lock attempt that isn't matched can prevent the lock + * from being freed. It'll block waiting to send its request for its + * mode which will prevent the lock from being freed when the null + * response arrives. + */ +static int scoutfs_lock_shrink(struct shrinker *shrink, + struct shrink_control *sc) +{ + struct lock_info *linfo = container_of(shrink, struct lock_info, + shrinker); + struct super_block *sb = linfo->sb; + struct scoutfs_lock *lock; + struct scoutfs_lock *tmp; + unsigned long nr; + bool added = false; + int ret; + + nr = sc->nr_to_scan; + if (nr == 0) + goto out; + + spin_lock(&linfo->lock); + +restart: + list_for_each_entry_safe(lock, tmp, &linfo->lru_list, lru_head) { + + BUG_ON(!lock_idle(lock)); + BUG_ON(lock->mode == SCOUTFS_LOCK_NULL); + BUG_ON(!list_empty(&lock->shrink_head)); + + if (linfo->shutdown || nr-- == 0) + break; + + __lock_del_lru(linfo, lock); + lock->request_pending = 1; + list_add_tail(&lock->shrink_head, &linfo->shrink_list); + added = true; + + scoutfs_inc_counter(sb, lock_shrink_attempted); + trace_scoutfs_lock_shrink(sb, lock); + + /* could have bazillions of idle locks */ + if (cond_resched_lock(&linfo->lock)) + goto restart; + } + + spin_unlock(&linfo->lock); + + if (added) + queue_work(linfo->workq, &linfo->shrink_work); + +out: + ret = min_t(unsigned long, linfo->lru_nr, INT_MAX); + trace_scoutfs_lock_shrink_exit(sb, sc->nr_to_scan, ret); + return ret; +} + +void scoutfs_free_unused_locks(struct super_block *sb, unsigned long nr) +{ + struct lock_info *linfo = SCOUTFS_SB(sb)->lock_info; + struct shrink_control sc = { + .gfp_mask = GFP_NOFS, + .nr_to_scan = INT_MAX, + }; + + linfo->shrinker.shrink(&linfo->shrinker, &sc); +} + +static void lock_tseq_show(struct seq_file *m, struct scoutfs_tseq_entry *ent) +{ + struct scoutfs_lock *lock = + container_of(ent, struct scoutfs_lock, tseq_entry); + + seq_printf(m, "start "SK_FMT" end "SK_FMT" refresh_gen %llu mode %d waiters: rd %u wr %u wo %u users: rd %u wr %u wo %u\n", + SK_ARG(&lock->start), SK_ARG(&lock->end), + lock->refresh_gen, lock->mode, + lock->waiters[SCOUTFS_LOCK_READ], + lock->waiters[SCOUTFS_LOCK_WRITE], + lock->waiters[SCOUTFS_LOCK_WRITE_ONLY], + lock->users[SCOUTFS_LOCK_READ], + lock->users[SCOUTFS_LOCK_WRITE], + lock->users[SCOUTFS_LOCK_WRITE_ONLY]); +} + +/* + * The caller is going to be calling _destroy soon and, critically, is + * about to shutdown networking before calling us so that we don't get + * any callbacks while we're destroying. We have to ensure that we + * won't call networking after this returns. + * + * Internal fs threads can be using locking, and locking can have async + * work pending. We use ->shutdown to force callers to return + * -ESHUTDOWN and to prevent the future queueing of work that could call + * networking. Locks whose work is stopped will be torn down by _destroy. + */ +void scoutfs_lock_shutdown(struct super_block *sb) +{ + DECLARE_LOCK_INFO(sb, linfo); + struct scoutfs_lock *lock; + struct rb_node *node; + + if (!linfo) + return; + + trace_scoutfs_lock_shutdown(sb, linfo); + + spin_lock(&linfo->lock); + + linfo->shutdown = true; + for (node = rb_first(&linfo->lock_tree); node; node = rb_next(node)) { + lock = rb_entry(node, struct scoutfs_lock, node); + wake_up(&lock->waitq); + } + + spin_unlock(&linfo->lock); + + flush_work(&linfo->grant_work); + flush_delayed_work(&linfo->inv_dwork); + flush_work(&linfo->shrink_work); +} + +/* + * By the time we get here the caller should have called _shutdown() and + * then called into all the subsystems that held locks to drop them. + * There should be no active users of locks and all future lock calls + * should fail. + * + * The client networking connection will have been shutdown so we don't + * get any request or response processing calls. + * + * Our job is to make sure nothing references the remaining locks and + * free them. + */ +void scoutfs_lock_destroy(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + DECLARE_LOCK_INFO(sb, linfo); + struct scoutfs_lock *lock; + struct rb_node *node; + enum scoutfs_lock_mode mode; + + if (!linfo) + return; + + trace_scoutfs_lock_destroy(sb, linfo); + + /* stop the shrinker from queueing work */ + unregister_shrinker(&linfo->shrinker); + + /* make sure that no one's actively using locks */ + spin_lock(&linfo->lock); + for (node = rb_first(&linfo->lock_tree); node; node = rb_next(node)) { + lock = rb_entry(node, struct scoutfs_lock, node); + + for (mode = 0; mode < SCOUTFS_LOCK_NR_MODES; mode++) { + if (lock->waiters[mode] || lock->users[mode]) { + scoutfs_warn(sb, "lock start "SK_FMT" end "SK_FMT" has mode %d user after shutdown", + SK_ARG(&lock->start), + SK_ARG(&lock->end), mode); + break; + } + } + } + spin_unlock(&linfo->lock); + + if (linfo->workq) { + /* pending grace work queues normal work */ + flush_workqueue(linfo->workq); + /* now all work won't queue itself */ + destroy_workqueue(linfo->workq); + } + + /* XXX does anything synchronize with open debugfs fds? */ + debugfs_remove(linfo->tseq_dentry); + + /* + * Usually lock_free is only called once locks are idle but all + * locks are idle by definition during shutdown. We need to + * manually update the lock's state to reflect that we've given + * up on pending work that would otherwise prevent free from + * being called (and would trip assertions in our manual calling + * of free). + */ + spin_lock(&linfo->lock); + node = rb_first(&linfo->lock_tree); + while (node) { + lock = rb_entry(node, struct scoutfs_lock, node); + node = rb_next(node); + lock->request_pending = 0; + if (!list_empty(&lock->lru_head)) + __lock_del_lru(linfo, lock); + if (!list_empty(&lock->grant_head)) + list_del_init(&lock->grant_head); + if (!list_empty(&lock->inv_head)) + list_del_init(&lock->inv_head); + if (!list_empty(&lock->shrink_head)) + list_del_init(&lock->shrink_head); + lock_remove(linfo, lock); + lock_free(linfo, lock); + } + spin_unlock(&linfo->lock); + + kfree(linfo); + sbi->lock_info = NULL; +} + +int scoutfs_lock_setup(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct lock_info *linfo; + int ret; + + linfo = kzalloc(sizeof(struct lock_info), GFP_KERNEL); + if (!linfo) + return -ENOMEM; + + linfo->sb = sb; + spin_lock_init(&linfo->lock); + linfo->lock_tree = RB_ROOT; + linfo->lock_range_tree = RB_ROOT; + linfo->shrinker.shrink = scoutfs_lock_shrink; + linfo->shrinker.seeks = DEFAULT_SEEKS; + register_shrinker(&linfo->shrinker); + INIT_LIST_HEAD(&linfo->lru_list); + INIT_WORK(&linfo->grant_work, lock_grant_worker); + INIT_LIST_HEAD(&linfo->grant_list); + INIT_DELAYED_WORK(&linfo->inv_dwork, lock_invalidate_worker); + INIT_LIST_HEAD(&linfo->inv_list); + INIT_WORK(&linfo->shrink_work, lock_shrink_worker); + INIT_LIST_HEAD(&linfo->shrink_list); + atomic64_set(&linfo->next_refresh_gen, 0); + scoutfs_tseq_tree_init(&linfo->tseq_tree, lock_tseq_show); + + sbi->lock_info = linfo; + trace_scoutfs_lock_setup(sb, linfo); + + linfo->tseq_dentry = scoutfs_tseq_create("client_locks", + sbi->debug_root, + &linfo->tseq_tree); + if (!linfo->tseq_dentry) { + ret = -ENOMEM; + goto out; + } + + linfo->workq = alloc_workqueue("scoutfs_lock_client_work", + WQ_NON_REENTRANT | WQ_UNBOUND | + WQ_HIGHPRI, 0); + if (!linfo->workq) { + ret = -ENOMEM; + goto out; + } + + ret = 0; +out: + if (ret) + scoutfs_lock_destroy(sb); + + return ret; +} diff --git a/kmod/src/lock.h b/kmod/src/lock.h new file mode 100644 index 00000000..b447df54 --- /dev/null +++ b/kmod/src/lock.h @@ -0,0 +1,105 @@ +#ifndef _SCOUTFS_LOCK_H_ +#define _SCOUTFS_LOCK_H_ + +#include "key.h" +#include "tseq.h" + +#define SCOUTFS_LKF_REFRESH_INODE 0x01 /* update stale inode from item */ +#define SCOUTFS_LKF_NONBLOCK 0x02 /* only use already held locks */ +#define SCOUTFS_LKF_INVALID (~((SCOUTFS_LKF_NONBLOCK << 1) - 1)) + +#define SCOUTFS_LOCK_NR_MODES SCOUTFS_LOCK_INVALID + +/* + * A few fields (start, end, refresh_gen, write_version, granted_mode) + * are referenced by code outside lock.c. + */ +struct scoutfs_lock { + struct super_block *sb; + struct scoutfs_key start; + struct scoutfs_key end; + struct rb_node node; + struct rb_node range_node; + u64 refresh_gen; + u64 write_version; + u64 dirty_trans_seq; + struct scoutfs_net_roots roots; + struct list_head lru_head; + wait_queue_head_t waitq; + ktime_t grace_deadline; + unsigned long request_pending:1, + invalidate_pending:1; + + struct list_head grant_head; + struct scoutfs_net_lock_grant_response grant_resp; + struct list_head inv_head; + struct scoutfs_net_lock inv_nl; + u64 inv_net_id; + struct list_head shrink_head; + + spinlock_t cov_list_lock; + struct list_head cov_list; + + enum scoutfs_lock_mode mode; + unsigned int waiters[SCOUTFS_LOCK_NR_MODES]; + unsigned int users[SCOUTFS_LOCK_NR_MODES]; + + struct scoutfs_tseq_entry tseq_entry; + + /* the forest tracks which log tree last saw bloom bit updates */ + atomic64_t forest_bloom_nr; +}; + +struct scoutfs_lock_coverage { + spinlock_t cov_lock; + struct scoutfs_lock *lock; + struct list_head head; +}; + +int scoutfs_lock_grant_response(struct super_block *sb, + struct scoutfs_net_lock_grant_response *gr); +int scoutfs_lock_invalidate_request(struct super_block *sb, u64 net_id, + struct scoutfs_net_lock *nl); +int scoutfs_lock_recover_request(struct super_block *sb, u64 net_id, + struct scoutfs_key *key); + +int scoutfs_lock_inode(struct super_block *sb, enum scoutfs_lock_mode mode, int flags, + struct inode *inode, struct scoutfs_lock **ret_lock); +int scoutfs_lock_ino(struct super_block *sb, enum scoutfs_lock_mode mode, int flags, u64 ino, + struct scoutfs_lock **ret_lock); +void scoutfs_lock_get_index_item_range(u8 type, u64 major, u64 ino, + struct scoutfs_key *start, + struct scoutfs_key *end); +int scoutfs_lock_inode_index(struct super_block *sb, enum scoutfs_lock_mode mode, + u8 type, u64 major, u64 ino, + struct scoutfs_lock **ret_lock); +int scoutfs_lock_inodes(struct super_block *sb, enum scoutfs_lock_mode mode, int flags, + struct inode *a, struct scoutfs_lock **a_lock, + struct inode *b, struct scoutfs_lock **b_lock, + struct inode *c, struct scoutfs_lock **c_lock, + struct inode *d, struct scoutfs_lock **D_lock); +int scoutfs_lock_rename(struct super_block *sb, enum scoutfs_lock_mode mode, int flags, + struct scoutfs_lock **lock); +int scoutfs_lock_rid(struct super_block *sb, enum scoutfs_lock_mode mode, int flags, + u64 rid, struct scoutfs_lock **lock); +void scoutfs_unlock(struct super_block *sb, struct scoutfs_lock *lock, + enum scoutfs_lock_mode mode); + +void scoutfs_lock_init_coverage(struct scoutfs_lock_coverage *cov); +void scoutfs_lock_add_coverage(struct super_block *sb, + struct scoutfs_lock *lock, + struct scoutfs_lock_coverage *cov); +bool scoutfs_lock_is_covered(struct super_block *sb, + struct scoutfs_lock_coverage *cov); +void scoutfs_lock_del_coverage(struct super_block *sb, + struct scoutfs_lock_coverage *cov); +bool scoutfs_lock_protected(struct scoutfs_lock *lock, struct scoutfs_key *key, + enum scoutfs_lock_mode mode); + +void scoutfs_free_unused_locks(struct super_block *sb, unsigned long nr); + +int scoutfs_lock_setup(struct super_block *sb); +void scoutfs_lock_shutdown(struct super_block *sb); +void scoutfs_lock_destroy(struct super_block *sb); + +#endif diff --git a/kmod/src/lock_server.c b/kmod/src/lock_server.c new file mode 100644 index 00000000..036bfcc4 --- /dev/null +++ b/kmod/src/lock_server.c @@ -0,0 +1,1074 @@ +/* + * Copyright (C) 2019 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include + +#include "format.h" +#include "counters.h" +#include "net.h" +#include "tseq.h" +#include "spbm.h" +#include "block.h" +#include "btree.h" +#include "msg.h" +#include "scoutfs_trace.h" +#include "lock_server.h" + +/* + * The scoutfs server implements a simple lock service. Client mounts + * request access to locks identified by a key. The server ensures that + * access mode exclusion is properly enforced. + * + * The server processing paths are implemented in network message + * receive processing callbacks. We're receiving either a grant request + * or an invalidation response. These processing callbacks are fully + * concurrent. Our grant responses and invalidation requests are sent + * from these contexts. + * + * We separate the locking of the global index of tracked locks from the + * locking of a lock's state. This allows concurrent work on unrelated + * locks and lets processing block sending responses to unresponsive + * clients without affecting other locks. + * + * Correctness of the protocol relies on the client and server each only + * sending one request at a time for a given lock. The server won't + * process a request from a client until its outstanding invalidation + * requests for the lock to other clients have been completed. The + * server specifies both the old mode and new mode when sending messages + * to the client. This lets the client resolve possible reordering when + * processing incoming grant responses and invalidation requests. The + * server doesn't use the modes specified by the clients but they're + * provided to add context. + * + * The server relies on the client's static rid and on reliable + * messaging. Each client has a rid that is unique for its life time. + * Message requests and responses are reliably delivered in order across + * reconnection. + * + * The server maintains a persistent record of connected clients. A new + * server instance discovers these and waits for previously connected + * clients to reconnect and recover their state before proceeding. If + * clients don't reconnect they are forcefully prevented from unsafely + * accessing the shared persistent storage. (fenced, according to the + * rules of the platform.. could range from being powered off to having + * their switch port disabled to having their local block device set + * read-only.) + * + * The lock server doesn't respond to memory pressure. The only way + * locks are freed is if they are invalidated to null on behalf of a + * conflicting request, clients specifically request a null mode, or the + * server shuts down. + */ + +#define LOCK_SERVER_RECOVERY_MS (10 * MSEC_PER_SEC) + +struct lock_server_info { + struct super_block *sb; + + spinlock_t lock; + struct mutex mutex; + struct rb_root locks_root; + + struct scoutfs_spbm recovery_pending; + struct delayed_work recovery_dwork; + + struct scoutfs_tseq_tree tseq_tree; + struct dentry *tseq_dentry; + + struct scoutfs_alloc *alloc; + struct scoutfs_block_writer *wri; + + atomic64_t write_version; +}; + +#define DECLARE_LOCK_SERVER_INFO(sb, name) \ + struct lock_server_info *name = SCOUTFS_SB(sb)->lock_server_info + +/* + * The state of a lock on the server is a function of the state of the + * locks on all clients. + * + * @granted: + * granted or trigger invalidation of previously granted. + * The state of a lock on the server is a function of messages that have + * been sent and received from clients on behalf of a given lock. + * + * While the invalidated list has entries, which means invalidation + * messages are still in flight, no more requests will be processed. + */ +struct server_lock_node { + atomic_t refcount; + struct mutex mutex; + struct rb_node node; + struct scoutfs_key key; + + struct list_head granted; + struct list_head requested; + struct list_head invalidated; +}; + +/* + * Interactions with the client are tracked with these little mode + * wrappers. + * + * @entry: The client mode's entry on one of the server lock lists indicating + * that the mode is actively granted, a pending request from the client, + * or a pending invalidation sent to the client. + * + * @rid: The client's rid used to send messages and tear down + * state as client's exit. + * + * @net_id: The id of a client's request used to send grant responses. The + * id of invalidation requests sent to clients that could be used to cancel + * the message. + * + * @mode: the mode that is granted to the client, that the client + * requested, or that the server is asserting with a pending + * invalidation request message. + */ +struct client_lock_entry { + struct list_head head; + u64 rid; + u64 net_id; + u8 mode; + + struct server_lock_node *snode; + struct scoutfs_tseq_entry tseq_entry; + u8 on_list; +}; + +enum { + OL_GRANTED = 0, + OL_REQUESTED, + OL_INVALIDATED, +}; + +/* + * Put an entry on a server lock's list while being careful to move or + * add the list head and while maintaining debugging info. + */ +static void add_client_entry(struct server_lock_node *snode, + struct list_head *list, + struct client_lock_entry *clent) +{ + WARN_ON_ONCE(!mutex_is_locked(&snode->mutex)); + + if (list_empty(&clent->head)) + list_add_tail(&clent->head, list); + else + list_move_tail(&clent->head, list); + + clent->on_list = list == &snode->granted ? OL_GRANTED : + list == &snode->requested ? OL_REQUESTED : + OL_INVALIDATED; +} + +static void free_client_entry(struct lock_server_info *inf, + struct server_lock_node *snode, + struct client_lock_entry *clent) +{ + WARN_ON_ONCE(!mutex_is_locked(&snode->mutex)); + + if (!list_empty(&clent->head)) + list_del_init(&clent->head); + scoutfs_tseq_del(&inf->tseq_tree, &clent->tseq_entry); + kfree(clent); +} + +static bool invalid_mode(u8 mode) +{ + return mode >= SCOUTFS_LOCK_INVALID; +} + +/* + * Return the mode that we should invalidate a granted lock down to + * given an incompatible requested mode. Usually we completely + * invalidate the items because incompatible requests have to be writers + * and our cache will then be stale, but the single exception is + * invalidating down to a read lock having held a write lock because the + * cache is still valid for reads after being written out. + */ +static u8 invalidation_mode(u8 granted, u8 requested) +{ + if (granted == SCOUTFS_LOCK_WRITE && requested == SCOUTFS_LOCK_READ) + return SCOUTFS_LOCK_READ; + + return SCOUTFS_LOCK_NULL; +} + +/* + * Return true of the client lock instances described by the entries can + * be granted at the same time. Typically this only means they're both + * modes that are compatible between nodes. In addition there's the + * special case where a read lock on a client is compatible with a write + * lock on the same client because the client's cache covered by the + * read lock is still valid if they get a write lock. + */ +static bool client_entries_compatible(struct client_lock_entry *granted, + struct client_lock_entry *requested) +{ + return (granted->mode == requested->mode && + (granted->mode == SCOUTFS_LOCK_READ || + granted->mode == SCOUTFS_LOCK_WRITE_ONLY)) || + (granted->rid == requested->rid && + granted->mode == SCOUTFS_LOCK_READ && + requested->mode == SCOUTFS_LOCK_WRITE); +} + +/* + * Get a locked server lock, possibly inserting the caller's allocated + * lock if we don't find one for the given key. The server lock's mutex + * is held on return and the caller must put the lock when they're done. + */ +static struct server_lock_node *get_server_lock(struct lock_server_info *inf, + struct scoutfs_key *key, + struct server_lock_node *ins, + bool or_next) +{ + struct rb_root *root = &inf->locks_root; + struct server_lock_node *ret = NULL; + struct server_lock_node *next = NULL; + struct server_lock_node *snode; + struct rb_node *parent = NULL; + struct rb_node **node; + int cmp; + + spin_lock(&inf->lock); + + node = &root->rb_node; + while (*node) { + parent = *node; + snode = container_of(*node, struct server_lock_node, node); + + cmp = scoutfs_key_compare(key, &snode->key); + if (cmp < 0) { + if (or_next) + next = snode; + node = &(*node)->rb_left; + } else if (cmp > 0) { + node = &(*node)->rb_right; + } else { + ret = snode; + break; + } + } + + if (ret == NULL && ins) { + rb_link_node(&ins->node, parent, node); + rb_insert_color(&ins->node, root); + ret = ins; + } + + if (ret == NULL && or_next && next) + ret = next; + + if (ret) + atomic_inc(&ret->refcount); + + spin_unlock(&inf->lock); + + if (ret) + mutex_lock(&ret->mutex); + + return ret; +} + +/* Get a server lock node, allocating if one doesn't exist. Caller must put. */ +static struct server_lock_node *alloc_server_lock(struct lock_server_info *inf, + struct scoutfs_key *key) +{ + struct server_lock_node *snode; + struct server_lock_node *ins; + + snode = get_server_lock(inf, key, NULL, false); + if (snode == NULL) { + ins = kzalloc(sizeof(struct server_lock_node), GFP_NOFS); + if (ins) { + atomic_set(&ins->refcount, 0); + mutex_init(&ins->mutex); + ins->key = *key; + INIT_LIST_HEAD(&ins->granted); + INIT_LIST_HEAD(&ins->requested); + INIT_LIST_HEAD(&ins->invalidated); + + snode = get_server_lock(inf, key, ins, false); + if (snode != ins) + kfree(ins); + } + } + + return snode; +} + +/* + * Finish with a server lock which has the mutex held, freeing it if + * it's empty and unused. + */ +static void put_server_lock(struct lock_server_info *inf, + struct server_lock_node *snode) +{ + bool should_free = false; + + BUG_ON(!mutex_is_locked(&snode->mutex)); + + if (atomic_dec_and_test(&snode->refcount) && + list_empty(&snode->granted) && + list_empty(&snode->requested) && + list_empty(&snode->invalidated)) { + spin_lock(&inf->lock); + rb_erase(&snode->node, &inf->locks_root); + spin_unlock(&inf->lock); + should_free = true; + } + + mutex_unlock(&snode->mutex); + + if (should_free) + kfree(snode); +} + +static struct client_lock_entry *find_entry(struct server_lock_node *snode, + struct list_head *list, + u64 rid) +{ + struct client_lock_entry *clent; + + WARN_ON_ONCE(!mutex_is_locked(&snode->mutex)); + + list_for_each_entry(clent, list, head) { + if (clent->rid == rid) + return clent; + } + + return NULL; +} + +static int process_waiting_requests(struct super_block *sb, + struct server_lock_node *snode); + +/* + * The server is receiving an incoming request from a client. We queue + * it on the lock and process it. + * + * XXX shut down if we get enomem? + */ +int scoutfs_lock_server_request(struct super_block *sb, u64 rid, + u64 net_id, struct scoutfs_net_lock *nl) +{ + DECLARE_LOCK_SERVER_INFO(sb, inf); + struct client_lock_entry *clent; + struct server_lock_node *snode; + int ret; + + trace_scoutfs_lock_message(sb, SLT_SERVER, SLT_GRANT, SLT_REQUEST, + rid, net_id, nl); + + if (invalid_mode(nl->old_mode) || invalid_mode(nl->new_mode)) { + ret = -EINVAL; + goto out; + } + + clent = kzalloc(sizeof(struct client_lock_entry), GFP_NOFS); + if (!clent) { + ret = -ENOMEM; + goto out; + } + + INIT_LIST_HEAD(&clent->head); + clent->rid = rid; + clent->net_id = net_id; + clent->mode = nl->new_mode; + + snode = alloc_server_lock(inf, &nl->key); + if (snode == NULL) { + kfree(clent); + ret = -ENOMEM; + goto out; + } + + clent->snode = snode; + add_client_entry(snode, &snode->requested, clent); + scoutfs_tseq_add(&inf->tseq_tree, &clent->tseq_entry); + + ret = process_waiting_requests(sb, snode); +out: + return ret; +} + +/* + * The server is receiving an invalidation response from the client. + * Find the client's entry on the server lock's invalidation list and + * free it so that request processing might be able to make forward + * progress. + * + * XXX what to do with errors? kick the client? + */ +int scoutfs_lock_server_response(struct super_block *sb, u64 rid, + struct scoutfs_net_lock *nl) +{ + DECLARE_LOCK_SERVER_INFO(sb, inf); + struct client_lock_entry *clent; + struct server_lock_node *snode; + int ret; + + trace_scoutfs_lock_message(sb, SLT_SERVER, SLT_INVALIDATE, SLT_RESPONSE, + rid, 0, nl); + + if (invalid_mode(nl->old_mode) || invalid_mode(nl->new_mode)) { + ret = -EINVAL; + goto out; + } + + /* XXX should always have a server lock here? recovery? */ + snode = get_server_lock(inf, &nl->key, NULL, false); + if (!snode) { + ret = -EINVAL; + goto out; + } + + clent = find_entry(snode, &snode->invalidated, rid); + if (!clent) { + put_server_lock(inf, snode); + ret = -EINVAL; + goto out; + } + + if (nl->new_mode == SCOUTFS_LOCK_NULL) { + free_client_entry(inf, snode, clent); + } else { + clent->mode = nl->new_mode; + add_client_entry(snode, &snode->granted, clent); + } + + ret = process_waiting_requests(sb, snode); +out: + return ret; +} + +/* + * Make forward progress on a lock by checking each waiting request in + * the order that they were received. If the next request is compatible + * with all the clients' grants then the request is granted and a + * response is sent. + * + * Invalidation requests are sent for every client grant that is + * incompatible with the next request. We won't process the next + * request again until we receive all the invalidation responses. Once + * they're all received then the request can be processed and will be + * compatible with the remaining grants. + * + * This is called with the snode mutex held. This can free the snode if + * it's empty. The caller can't reference the snode once this returns + * so we unlock the snode mutex. + * + * All progress must wait for all clients to finish with recovery + * because we don't know which locks they'll hold. The unlocked + * recovery_pending test here is OK. It's filled by setup before + * anything runs. It's emptied by recovery completion. We can get a + * false nonempty result if we race with recovery completion, but that's + * OK because recovery completion processes all the locks that have + * requests after emptying, including the unlikely loser of that race. + */ +static int process_waiting_requests(struct super_block *sb, + struct server_lock_node *snode) +{ + DECLARE_LOCK_SERVER_INFO(sb, inf); + struct scoutfs_net_lock_grant_response gres; + struct scoutfs_net_lock nl; + struct client_lock_entry *req; + struct client_lock_entry *req_tmp; + struct client_lock_entry *gr; + struct client_lock_entry *gr_tmp; + u64 wv; + int ret; + + BUG_ON(!mutex_is_locked(&snode->mutex)); + + /* processing waits for all invalidation responses or recovery */ + if (!list_empty(&snode->invalidated) || + !scoutfs_spbm_empty(&inf->recovery_pending)) { + ret = 0; + goto out; + } + + /* walk through pending requests in order received */ + list_for_each_entry_safe(req, req_tmp, &snode->requested, head) { + + /* send invalidation to any incompatible grants */ + list_for_each_entry_safe(gr, gr_tmp, &snode->granted, head) { + if (client_entries_compatible(gr, req)) + continue; + + nl.key = snode->key; + nl.old_mode = gr->mode; + nl.new_mode = invalidation_mode(gr->mode, req->mode); + + ret = scoutfs_server_lock_request(sb, gr->rid, &nl); + if (ret) + goto out; + + trace_scoutfs_lock_message(sb, SLT_SERVER, + SLT_INVALIDATE, SLT_REQUEST, + gr->rid, 0, &nl); + + add_client_entry(snode, &snode->invalidated, gr); + } + + /* wait for any newly sent invalidations */ + if (!list_empty(&snode->invalidated)) + break; + + nl.key = snode->key; + nl.new_mode = req->mode; + + /* see if there's an existing compatible grant to replace */ + gr = find_entry(snode, &snode->granted, req->rid); + if (gr) { + nl.old_mode = gr->mode; + free_client_entry(inf, snode, gr); + } else { + nl.old_mode = SCOUTFS_LOCK_NULL; + } + + if (nl.new_mode == SCOUTFS_LOCK_WRITE || + nl.new_mode == SCOUTFS_LOCK_WRITE_ONLY) { + wv = atomic64_inc_return(&inf->write_version); + nl.write_version = cpu_to_le64(wv); + } + + gres.nl = nl; + scoutfs_server_get_roots(sb, &gres.roots); + + ret = scoutfs_server_lock_response(sb, req->rid, + req->net_id, &gres); + if (ret) + goto out; + + trace_scoutfs_lock_message(sb, SLT_SERVER, SLT_GRANT, + SLT_RESPONSE, req->rid, + req->net_id, &nl); + + /* don't track null client locks, track all else */ + if (req->mode == SCOUTFS_LOCK_NULL) + free_client_entry(inf, snode, req); + else + add_client_entry(snode, &snode->granted, req); + } + + ret = 0; +out: + put_server_lock(inf, snode); + + return ret; +} + +static void init_lock_clients_key(struct scoutfs_key *key, u64 rid) +{ + *key = (struct scoutfs_key) { + .sk_zone = SCOUTFS_LOCK_CLIENTS_ZONE, + .sklc_rid = cpu_to_le64(rid), + }; +} + +/* + * The server received a greeting from a client for the first time. If + * the client had already talked to the server then we must find an + * existing record for it and should begin recovery. If it doesn't have + * a record then its timed out and we can't allow it to reconnect. If + * its connecting for the first time then we insert a new record. If + * + * This is running in concurrent client greeting processing contexts. + */ +int scoutfs_lock_server_greeting(struct super_block *sb, u64 rid, + bool should_exist) +{ + DECLARE_LOCK_SERVER_INFO(sb, inf); + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + SCOUTFS_BTREE_ITEM_REF(iref); + struct scoutfs_key key; + int ret; + + init_lock_clients_key(&key, rid); + + mutex_lock(&inf->mutex); + if (should_exist) { + ret = scoutfs_btree_lookup(sb, &super->lock_clients, &key, + &iref); + if (ret == 0) + scoutfs_btree_put_iref(&iref); + } else { + ret = scoutfs_btree_insert(sb, inf->alloc, inf->wri, + &super->lock_clients, + &key, NULL, 0); + } + mutex_unlock(&inf->mutex); + + if (should_exist && ret == 0) { + scoutfs_key_set_zeros(&key); + ret = scoutfs_server_lock_recover_request(sb, rid, &key); + if (ret) + goto out; + } + +out: + return ret; +} + +/* + * A client sent their last recovery response and can exit recovery. If + * they were the last client in recovery then we can process all the + * server locks that had requests. + */ +static int finished_recovery(struct super_block *sb, u64 rid, bool cancel) +{ + DECLARE_LOCK_SERVER_INFO(sb, inf); + struct server_lock_node *snode; + struct scoutfs_key key; + bool still_pending; + int ret = 0; + + spin_lock(&inf->lock); + scoutfs_spbm_clear(&inf->recovery_pending, rid); + still_pending = !scoutfs_spbm_empty(&inf->recovery_pending); + spin_unlock(&inf->lock); + if (still_pending) + return 0; + + if (cancel) + cancel_delayed_work_sync(&inf->recovery_dwork); + + scoutfs_key_set_zeros(&key); + + scoutfs_info(sb, "all lock clients recovered"); + + while ((snode = get_server_lock(inf, &key, NULL, true))) { + + key = snode->key; + scoutfs_key_inc(&key); + + if (!list_empty(&snode->requested)) { + ret = process_waiting_requests(sb, snode); + if (ret) + break; + } else { + put_server_lock(inf, snode); + } + } + + return ret; +} + +static void set_max_write_version(struct lock_server_info *inf, u64 new) +{ + u64 old; + + while (new > (old = atomic64_read(&inf->write_version)) && + (atomic64_cmpxchg(&inf->write_version, old, new) != old)); +} + +/* + * We sent a lock recover request to the client when we received its + * greeting while in recovery. Here we instantiate all the locks it + * gave us in response and send another request from the next key. + * We're done once we receive an empty response. + */ +int scoutfs_lock_server_recover_response(struct super_block *sb, u64 rid, + struct scoutfs_net_lock_recover *nlr) +{ + DECLARE_LOCK_SERVER_INFO(sb, inf); + struct client_lock_entry *existing; + struct client_lock_entry *clent; + struct server_lock_node *snode; + struct scoutfs_key key; + int ret = 0; + int i; + + /* client must be in recovery */ + spin_lock(&inf->lock); + if (!scoutfs_spbm_test(&inf->recovery_pending, rid)) + ret = -EINVAL; + spin_unlock(&inf->lock); + if (ret) + goto out; + + /* client has sent us all their locks */ + if (nlr->nr == 0) { + ret = finished_recovery(sb, rid, true); + goto out; + } + + for (i = 0; i < le16_to_cpu(nlr->nr); i++) { + clent = kzalloc(sizeof(struct client_lock_entry), GFP_NOFS); + if (!clent) { + ret = -ENOMEM; + goto out; + } + + INIT_LIST_HEAD(&clent->head); + clent->rid = rid; + clent->net_id = 0; + clent->mode = nlr->locks[i].new_mode; + + snode = alloc_server_lock(inf, &nlr->locks[i].key); + if (snode == NULL) { + kfree(clent); + ret = -ENOMEM; + goto out; + } + + existing = find_entry(snode, &snode->granted, rid); + if (existing) { + kfree(clent); + put_server_lock(inf, snode); + ret = -EEXIST; + goto out; + } + + clent->snode = snode; + add_client_entry(snode, &snode->granted, clent); + scoutfs_tseq_add(&inf->tseq_tree, &clent->tseq_entry); + + put_server_lock(inf, snode); + + /* make sure next write lock is greater than all recovered */ + set_max_write_version(inf, + le64_to_cpu(nlr->locks[i].write_version)); + } + + /* send request for next batch of keys */ + key = nlr->locks[le16_to_cpu(nlr->nr) - 1].key; + scoutfs_key_inc(&key); + + ret = scoutfs_server_lock_recover_request(sb, rid, &key); +out: + return ret; +} + +static int get_rid_and_put_ref(struct scoutfs_btree_item_ref *iref, u64 *rid) +{ + int ret; + + if (iref->val_len == 0) { + *rid = le64_to_cpu(iref->key->sklc_rid); + ret = 0; + } else { + ret = -EIO; + } + scoutfs_btree_put_iref(iref); + return ret; +} + +/* + * This work executes if enough time passes without all of the clients + * finishing with recovery and canceling the work. We walk through the + * client records and find any that still have their recovery pending. + */ +static void scoutfs_lock_server_recovery_timeout(struct work_struct *work) +{ + struct lock_server_info *inf = container_of(work, + struct lock_server_info, + recovery_dwork.work); + struct super_block *sb = inf->sb; + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + SCOUTFS_BTREE_ITEM_REF(iref); + struct scoutfs_key key; + bool timed_out; + u64 rid; + int ret; + + ret = scoutfs_server_hold_commit(sb); + if (ret) + goto out; + + /* we enter recovery if there are any client records */ + for (rid = 0; ; rid++) { + init_lock_clients_key(&key, rid); + ret = scoutfs_btree_next(sb, &super->lock_clients, &key, &iref); + if (ret == -ENOENT) { + ret = 0; + break; + } + if (ret == 0) + ret = get_rid_and_put_ref(&iref, &rid); + if (ret < 0) + break; + + spin_lock(&inf->lock); + if (scoutfs_spbm_test(&inf->recovery_pending, rid)) { + scoutfs_spbm_clear(&inf->recovery_pending, rid); + timed_out = true; + } else { + timed_out = false; + } + spin_unlock(&inf->lock); + + if (!timed_out) + continue; + + scoutfs_err(sb, "client rid %016llx lock recovery timed out", + rid); + + init_lock_clients_key(&key, rid); + ret = scoutfs_btree_delete(sb, inf->alloc, inf->wri, + &super->lock_clients, &key); + if (ret) + break; + } + + ret = scoutfs_server_apply_commit(sb, ret); +out: + /* force processing all pending lock requests */ + if (ret == 0) + ret = finished_recovery(sb, 0, false); + + if (ret < 0) { + scoutfs_err(sb, "lock server saw err %d while timing out clients, shutting down", ret); + scoutfs_server_abort(sb); + } +} + +/* + * A client is leaving the lock service. They aren't using locks and + * won't send any more requests. We tear down all the state we had for + * them. This can be called multiple times for a given client as their + * farewell is resent to new servers. It's OK to not find any state. + * If we fail to delete a persistent entry then we have to shut down and + * hope that the next server has more luck. + */ +int scoutfs_lock_server_farewell(struct super_block *sb, u64 rid) +{ + DECLARE_LOCK_SERVER_INFO(sb, inf); + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + struct client_lock_entry *clent; + struct client_lock_entry *tmp; + struct server_lock_node *snode; + struct scoutfs_key key; + struct list_head *list; + bool freed; + int ret = 0; + + mutex_lock(&inf->mutex); + init_lock_clients_key(&key, rid); + ret = scoutfs_btree_delete(sb, inf->alloc, inf->wri, + &super->lock_clients, &key); + mutex_unlock(&inf->mutex); + if (ret == -ENOENT) { + ret = 0; + goto out; + } + if (ret < 0) + goto out; + + scoutfs_key_set_zeros(&key); + + while ((snode = get_server_lock(inf, &key, NULL, true))) { + + freed = false; + for (list = &snode->granted; list != NULL; + list = (list == &snode->granted) ? &snode->requested : + (list == &snode->requested) ? &snode->invalidated : + NULL) { + + list_for_each_entry_safe(clent, tmp, list, head) { + if (clent->rid == rid) { + free_client_entry(inf, snode, clent); + freed = true; + } + } + } + + key = snode->key; + scoutfs_key_inc(&key); + + if (freed) { + ret = process_waiting_requests(sb, snode); + if (ret) + goto out; + } else { + put_server_lock(inf, snode); + } + } + ret = 0; + +out: + if (ret < 0) { + scoutfs_err(sb, "lock server err %d during client rid %016llx farewell, shutting down", + ret, rid); + scoutfs_server_abort(sb); + } + + return ret; +} + +static char *lock_mode_string(u8 mode) +{ + static char *mode_strings[] = { + [SCOUTFS_LOCK_NULL] = "null", + [SCOUTFS_LOCK_READ] = "read", + [SCOUTFS_LOCK_WRITE] = "write", + [SCOUTFS_LOCK_WRITE_ONLY] = "write_only", + }; + + if (mode < ARRAY_SIZE(mode_strings) && mode_strings[mode]) + return mode_strings[mode]; + + return "unknown"; +} + +static char *lock_on_list_string(u8 on_list) +{ + static char *on_list_strings[] = { + [OL_GRANTED] = "granted", + [OL_REQUESTED] = "requested", + [OL_INVALIDATED] = "invalidated", + }; + + if (on_list < ARRAY_SIZE(on_list_strings) && on_list_strings[on_list]) + return on_list_strings[on_list]; + + return "unknown"; +} + +static void lock_server_tseq_show(struct seq_file *m, + struct scoutfs_tseq_entry *ent) +{ + struct client_lock_entry *clent = container_of(ent, + struct client_lock_entry, + tseq_entry); + struct server_lock_node *snode = clent->snode; + + seq_printf(m, SK_FMT" %s %s rid %016llx net_id %llu\n", + SK_ARG(&snode->key), lock_mode_string(clent->mode), + lock_on_list_string(clent->on_list), clent->rid, + clent->net_id); +} + +/* + * Setup the lock server. This is called before networking can deliver + * requests. If we find existing client records then we enter recovery. + * Lock request processing is deferred until recovery is resolved for + * all the existing clients, either they reconnect and replay locks or + * we time them out. + */ +int scoutfs_lock_server_setup(struct super_block *sb, + struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, u64 max_vers) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + struct lock_server_info *inf; + SCOUTFS_BTREE_ITEM_REF(iref); + struct scoutfs_key key; + unsigned int nr; + u64 rid; + int ret; + + inf = kzalloc(sizeof(struct lock_server_info), GFP_KERNEL); + if (!inf) + return -ENOMEM; + + inf->sb = sb; + spin_lock_init(&inf->lock); + mutex_init(&inf->mutex); + inf->locks_root = RB_ROOT; + scoutfs_spbm_init(&inf->recovery_pending); + INIT_DELAYED_WORK(&inf->recovery_dwork, + scoutfs_lock_server_recovery_timeout); + scoutfs_tseq_tree_init(&inf->tseq_tree, lock_server_tseq_show); + inf->alloc = alloc; + inf->wri = wri; + atomic64_set(&inf->write_version, max_vers); /* inc_return gives +1 */ + + inf->tseq_dentry = scoutfs_tseq_create("server_locks", sbi->debug_root, + &inf->tseq_tree); + if (!inf->tseq_dentry) { + kfree(inf); + return -ENOMEM; + } + + sbi->lock_server_info = inf; + + /* we enter recovery if there are any client records */ + nr = 0; + for (rid = 0; ; rid++) { + init_lock_clients_key(&key, rid); + ret = scoutfs_btree_next(sb, &super->lock_clients, &key, &iref); + if (ret == -ENOENT) + break; + if (ret == 0) + ret = get_rid_and_put_ref(&iref, &rid); + if (ret < 0) + goto out; + + ret = scoutfs_spbm_set(&inf->recovery_pending, rid); + if (ret) + goto out; + nr++; + + if (rid == U64_MAX) + break; + } + ret = 0; + + if (nr) { + schedule_delayed_work(&inf->recovery_dwork, + msecs_to_jiffies(LOCK_SERVER_RECOVERY_MS)); + scoutfs_info(sb, "waiting for %u lock clients to recover", nr); + } + +out: + return ret; +} + +/* + * The server will have shut down networking before stopping us so we + * don't have to worry about message processing calls while we free. + */ +void scoutfs_lock_server_destroy(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + DECLARE_LOCK_SERVER_INFO(sb, inf); + struct server_lock_node *snode; + struct server_lock_node *stmp; + struct client_lock_entry *clent; + struct client_lock_entry *ctmp; + LIST_HEAD(list); + + if (inf) { + cancel_delayed_work_sync(&inf->recovery_dwork); + + debugfs_remove(inf->tseq_dentry); + + rbtree_postorder_for_each_entry_safe(snode, stmp, + &inf->locks_root, node) { + + list_splice_init(&snode->granted, &list); + list_splice_init(&snode->requested, &list); + list_splice_init(&snode->invalidated, &list); + + mutex_lock(&snode->mutex); + list_for_each_entry_safe(clent, ctmp, &list, head) { + free_client_entry(inf, snode, clent); + } + mutex_unlock(&snode->mutex); + + kfree(snode); + } + + scoutfs_spbm_destroy(&inf->recovery_pending); + + kfree(inf); + sbi->lock_server_info = NULL; + } +} diff --git a/kmod/src/lock_server.h b/kmod/src/lock_server.h new file mode 100644 index 00000000..357fd5af --- /dev/null +++ b/kmod/src/lock_server.h @@ -0,0 +1,19 @@ +#ifndef _SCOUTFS_LOCK_SERVER_H_ +#define _SCOUTFS_LOCK_SERVER_H_ + +int scoutfs_lock_server_recover_response(struct super_block *sb, u64 rid, + struct scoutfs_net_lock_recover *nlr); +int scoutfs_lock_server_request(struct super_block *sb, u64 rid, + u64 net_id, struct scoutfs_net_lock *nl); +int scoutfs_lock_server_greeting(struct super_block *sb, u64 rid, + bool should_exist); +int scoutfs_lock_server_response(struct super_block *sb, u64 rid, + struct scoutfs_net_lock *nl); +int scoutfs_lock_server_farewell(struct super_block *sb, u64 rid); + +int scoutfs_lock_server_setup(struct super_block *sb, + struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, u64 max_vers); +void scoutfs_lock_server_destroy(struct super_block *sb); + +#endif diff --git a/kmod/src/msg.c b/kmod/src/msg.c new file mode 100644 index 00000000..1d268de3 --- /dev/null +++ b/kmod/src/msg.c @@ -0,0 +1,24 @@ +#include +#include + +#include "msg.h" + +/* + * This can be called with pre-emption disabled if the caller is printing + * the contents of formated per-cpu key string buffers. + */ +void scoutfs_msg(struct super_block *sb, const char *prefix, const char *str, + const char *fmt, ...) +{ + struct va_format vaf; + va_list args; + + va_start(args, fmt); + + vaf.fmt = fmt; + vaf.va = &args; + + printk("%sscoutfs "SCSBF"%s: %pV\n", prefix, SCSB_ARGS(sb), str, &vaf); + + va_end(args); +} diff --git a/kmod/src/msg.h b/kmod/src/msg.h new file mode 100644 index 00000000..dbd33fb2 --- /dev/null +++ b/kmod/src/msg.h @@ -0,0 +1,55 @@ +#ifndef _SCOUTFS_MSG_H_ +#define _SCOUTFS_MSG_H_ + +#include +#include "key.h" +#include "counters.h" + +void __printf(4, 5) scoutfs_msg(struct super_block *sb, const char *prefix, + const char *str, const char *fmt, ...); + +#define scoutfs_msg_check(sb, pref, str, fmt, args...) \ +do { \ + BUILD_BUG_ON(fmt[sizeof(fmt) - 2] == '\n'); \ + scoutfs_msg(sb, pref, str, fmt, ##args); \ +} while (0) + +#define scoutfs_err(sb, fmt, args...) \ + scoutfs_msg_check(sb, KERN_ERR, " error", fmt, ##args) + +#define scoutfs_warn(sb, fmt, args...) \ + scoutfs_msg_check(sb, KERN_WARNING, " warning", fmt, ##args) + +#define scoutfs_info(sb, fmt, args...) \ + scoutfs_msg_check(sb, KERN_INFO, "", fmt, ##args) + +#define scoutfs_bug_on(sb, cond, fmt, args...) \ +do { \ + if (cond) { \ + scoutfs_err(sb, "(" __stringify(cond) "), " fmt, ##args); \ + BUG(); \ + } \ +} while (0) \ + +/* + * Each message is only generated once per volume. Remounting resets + * the messages. + */ +#define scoutfs_corruption(sb, which, counter, fmt, args...) \ +do { \ + __typeof__(sb) _sb = (sb); \ + struct scoutfs_sb_info *_sbi = SCOUTFS_SB(_sb); \ + unsigned int _bit = (which); \ + \ + if (WARN_ON_ONCE(_bit >= SC_NR_SOURCES)) \ + break; \ + \ + scoutfs_inc_counter(_sb, counter); \ + if (!test_and_set_bit(_bit, _sbi->corruption_messages_once)) { \ + scoutfs_err(_sb, "corruption (see scoutfs-corruption(5)): " \ + #which ": " fmt, ##args); \ + dump_stack(); \ + } \ +} while (0) \ + +#endif diff --git a/kmod/src/net.c b/kmod/src/net.c new file mode 100644 index 00000000..c885d2bd --- /dev/null +++ b/kmod/src/net.c @@ -0,0 +1,1913 @@ +/* + * Copyright (C) 2018 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "format.h" +#include "counters.h" +#include "inode.h" +#include "btree.h" +#include "scoutfs_trace.h" +#include "msg.h" +#include "net.h" +#include "endian_swap.h" +#include "tseq.h" + +/* + * scoutfs networking delivers requests and responses between nodes. + * + * Nodes decide to be either a connecting client or a listening server. + * Both set up a connection and specify the set of request commands they + * can process. + * + * Request and response messages are queued on a connection. They're + * resent down newly established sockets on a long lived connection. + * Queued requests are removed as a response is processed or if the + * request is canceled by the sender. Queued responses are removed as + * the receiver acknowledges their delivery. + * + * Request and response resending is asymmetrical because of the + * client/server relationship. If a client connects to a new server it + * drops responses because the new server doesn't have any requests + * pending. If a server times out a client it drops everything because + * that client is never coming back. + * + * Requests and responses are only processed once for a given client and + * server pair. Callers have to deal with the possibility that two + * servers might both process the same client request, even though the + * client may only see the most recent response. + * + * The functional core of this implementation is solid, but boy are the + * interface boundaries getting fuzzy. The core knows too much about + * clients and servers and the communications across the net interface + * boundary are questionable. We probably want to pull more client and + * server specific behaviour up into the client and server and turn the + * "net" code into more passive shared framing helpers. + * + * XXX: + * - trace command and response data payloads + * - checksum message contents? + * - shutdown server if accept can't alloc resources for new conn? + */ + +/* reasonable multiple of max client reconnect attempt interval */ +#define CLIENT_RECONNECT_TIMEOUT_MS (20 * MSEC_PER_SEC) + +/* + * A connection's shutdown work executes in its own workqueue so that the + * work can free the connection's workq. + */ +struct net_info { + struct workqueue_struct *shutdown_workq; + struct workqueue_struct *destroy_workq; + struct dentry *conn_tseq_dentry; + struct scoutfs_tseq_tree conn_tseq_tree; + struct dentry *msg_tseq_dentry; + struct scoutfs_tseq_tree msg_tseq_tree; +}; + +/* flags enum is in net.h */ +#define test_conn_fl(conn, which) (!!((conn)->flags & CONN_FL_##which)) +#define set_conn_fl(conn, which) \ +do { \ + (conn)->flags |= CONN_FL_##which; \ +} while (0) +#define clear_conn_fl(conn, which) \ +do { \ + (conn)->flags &= ~CONN_FL_##which; \ +} while (0) +#define assign_conn_fl(dst, src, which) \ +do { \ + (dst)->flags |= ((conn)->flags & CONN_FL_##which); \ +} while (0) + +/* listening and their accepting sockets have a fixed locking order */ +enum spin_lock_subtype { + CONN_LOCK_LISTENER, + CONN_LOCK_ACCEPTED, +}; + +/* + * Messages to be sent are allocated and put on the send queue. + * + * Request messages are put on the resend queue until their response + * messages is received and they can be freed. + * + * The send worker is the only context that references messages while + * not holding the lock. It does this while blocking sending the + * message down the socket. To free messages we mark them dead and have + * the send worker free them while under the lock so that we don't have + * to risk freeing messages from under the unlocked send worker. + */ +struct message_send { + struct scoutfs_tseq_entry tseq_entry; + unsigned long dead:1; + struct list_head head; + scoutfs_net_response_t resp_func; + void *resp_data; + struct scoutfs_net_header nh; +}; + +/* + * Incoming received messages are processed in concurrent blocking work + * contexts. + */ +struct message_recv { + struct scoutfs_tseq_entry tseq_entry; + struct work_struct proc_work; + struct scoutfs_net_connection *conn; + struct scoutfs_net_header nh; +}; + +#define DEFINE_CONN_FROM_WORK(name, work, member) \ + struct scoutfs_net_connection *name = \ + container_of(work, struct scoutfs_net_connection, member) + +/* Total message bytes including header and payload */ +static int nh_bytes(unsigned int data_len) +{ + return offsetof(struct scoutfs_net_header, data[data_len]); +} + +static bool nh_is_response(struct scoutfs_net_header *nh) +{ + return !!(nh->flags & SCOUTFS_NET_FLAG_RESPONSE); +} + +static bool nh_is_request(struct scoutfs_net_header *nh) +{ + return !nh_is_response(nh); +} + +/* + * We return dead requests so that the caller can stop searching other + * lists for the dead request that we found. + */ +static struct message_send *search_list(struct scoutfs_net_connection *conn, + struct list_head *list, + u8 cmd, u64 id) +{ + struct message_send *msend; + + assert_spin_locked(&conn->lock); + + list_for_each_entry(msend, list, head) { + if (nh_is_request(&msend->nh) && msend->nh.cmd == cmd && + le64_to_cpu(msend->nh.id) == id) + return msend; + } + + return NULL; +} + +/* + * Find an active send request on the lists. It's almost certainly + * waiting on the resend queue but it could be actively being sent. + */ +static struct message_send *find_request(struct scoutfs_net_connection *conn, + u8 cmd, u64 id) +{ + struct message_send *msend; + + msend = search_list(conn, &conn->resend_queue, cmd, id) ?: + search_list(conn, &conn->send_queue, cmd, id); + if (msend && msend->dead) + msend = NULL; + return msend; +} + +/* + * Complete a send message by moving it to the send queue and marking it + * to be freed. It won't be visible to callers trying to find sends. + */ +static void complete_send(struct scoutfs_net_connection *conn, + struct message_send *msend) +{ + assert_spin_locked(&conn->lock); + + if (WARN_ON_ONCE(msend->dead) || + WARN_ON_ONCE(list_empty(&msend->head))) + return; + + msend->dead = 1; + list_move(&msend->head, &conn->send_queue); + queue_work(conn->workq, &conn->send_work); +} + +/* + * Translate a positive error on the wire to a negative host errno. + */ +static inline int net_err_to_host(u8 net_err) +{ +#undef EXPAND_NET_ERRNO +#define EXPAND_NET_ERRNO(which) [SCOUTFS_NET_ERR_##which] = which, + static u8 host_errnos[] = { + EXPAND_EACH_NET_ERRNO + }; + + if (net_err == SCOUTFS_NET_ERR_NONE) + return 0; + + if (net_err < ARRAY_SIZE(host_errnos) && host_errnos[net_err]) + return -host_errnos[net_err]; + + return -EINVAL; +} + +/* + * Translate a negative host errno to a positive error on the wire. + * + * The caller is our kernel run time which should have been careful with + * errnos. But mistakes happen so let's holler and translate unknown + * errors. A fun bit of trivia: sparse's array bounds detection once + * got confused by conditions in WARN_ON_ONCE(); + */ +static inline u8 net_err_from_host(struct super_block *sb, int error) +{ +#undef EXPAND_NET_ERRNO +#define EXPAND_NET_ERRNO(which) [which] = SCOUTFS_NET_ERR_##which, + static u8 net_errs[] = { + EXPAND_EACH_NET_ERRNO + }; + int ind = -error; + + if (error == 0) + return SCOUTFS_NET_ERR_NONE; + + if (error > 0 || ind >= ARRAY_SIZE(net_errs) || net_errs[ind] == 0) { + static bool warned; + if (!warned) { + warned = 1; + scoutfs_warn(sb, "host errno %d sent as EINVAL", + error); + } + + return -EINVAL; + } + + return net_errs[ind]; +} + +/* + * Shutdown the connection. This is called by many contexts including + * work that most complete to finish shutting down. We queue specific + * shutdown work that can wait on all the connection's other work. + * We're sure to only queue the shutdown work once. + */ +static void shutdown_conn_locked(struct scoutfs_net_connection *conn) +{ + struct super_block *sb = conn->sb; + struct net_info *ninf = SCOUTFS_SB(sb)->net_info; + + assert_spin_locked(&conn->lock); + + if (!test_conn_fl(conn, shutting_down)) { + clear_conn_fl(conn, established); + set_conn_fl(conn, shutting_down); + trace_scoutfs_conn_shutdown_queued(conn); + queue_work(ninf->shutdown_workq, &conn->shutdown_work); + } +} + +static void shutdown_conn(struct scoutfs_net_connection *conn) +{ + spin_lock(&conn->lock); + shutdown_conn_locked(conn); + spin_unlock(&conn->lock); +} + +/* + * Allocate a message and put it on the send queue. + * + * A 0 id means that we'll assign the next id from the connection once + * we hold the lock and is only valid for sending requests. + * + * This can race with connections that are either starting up and + * shutting down. We only directly queue the send work if the + * connection has passed the greeting and isn't being shut down. At all + * other times we add new sends to the resend queue. + * + * If a non-zero rid is specified then the conn argument is a listening + * connection and the connection to send the message down is found by + * searching for the rid in its accepted connections. + */ +static int submit_send(struct super_block *sb, + struct scoutfs_net_connection *conn, u64 rid, + u8 cmd, u8 flags, u64 id, u8 net_err, + void *data, u16 data_len, + scoutfs_net_response_t resp_func, void *resp_data, + u64 *id_ret) +{ + struct net_info *ninf = SCOUTFS_SB(sb)->net_info; + struct scoutfs_net_connection *acc_conn; + struct message_send *msend; + u64 seq; + + if (WARN_ON_ONCE(cmd >= SCOUTFS_NET_CMD_UNKNOWN) || + WARN_ON_ONCE(flags & SCOUTFS_NET_FLAGS_UNKNOWN) || + WARN_ON_ONCE(net_err >= SCOUTFS_NET_ERR_UNKNOWN) || + WARN_ON_ONCE(data_len > SCOUTFS_NET_MAX_DATA_LEN) || + WARN_ON_ONCE(data_len && data == NULL) || + WARN_ON_ONCE(net_err && (!(flags & SCOUTFS_NET_FLAG_RESPONSE))) || + WARN_ON_ONCE(id == 0 && (flags & SCOUTFS_NET_FLAG_RESPONSE))) + return -EINVAL; + + msend = kmalloc(offsetof(struct message_send, + nh.data[data_len]), GFP_NOFS); + if (!msend) + return -ENOMEM; + + spin_lock_nested(&conn->lock, CONN_LOCK_LISTENER); + + if (rid != 0) { + list_for_each_entry(acc_conn, &conn->accepted_list, + accepted_head) { + if (acc_conn->rid == rid) { + spin_lock_nested(&acc_conn->lock, + CONN_LOCK_ACCEPTED); + spin_unlock(&conn->lock); + conn = acc_conn; + rid = 0; + break; + } + } + if (rid != 0) { + spin_unlock(&conn->lock); + return -ENOTCONN; + } + } + + seq = conn->next_send_seq++; + if (id == 0) + id = conn->next_send_id++; + + msend->resp_func = resp_func; + msend->resp_data = resp_data; + msend->dead = 0; + + msend->nh.seq = cpu_to_le64(seq); + msend->nh.recv_seq = 0; /* set when sent, not when queued */ + msend->nh.id = cpu_to_le64(id); + msend->nh.cmd = cmd; + msend->nh.flags = flags; + msend->nh.error = net_err; + memset(msend->nh.__pad, 0, sizeof(msend->nh.__pad)); + msend->nh.data_len = cpu_to_le16(data_len); + if (data_len) + memcpy(msend->nh.data, data, data_len); + + if (test_conn_fl(conn, established) && + (test_conn_fl(conn, valid_greeting) || + cmd == SCOUTFS_NET_CMD_GREETING)) { + list_add_tail(&msend->head, &conn->send_queue); + queue_work(conn->workq, &conn->send_work); + } else { + list_add_tail(&msend->head, &conn->resend_queue); + } + + if (id_ret) + *id_ret = le64_to_cpu(msend->nh.id); + + scoutfs_tseq_add(&ninf->msg_tseq_tree, &msend->tseq_entry); + + spin_unlock(&conn->lock); + + return 0; +} + +/* + * Process an incoming request. The greeting should ensure that the + * sender won't send us unknown commands. We return an error if we see + * an unknown command because the greeting should agree on an understood + * protocol. The request function sends a response and returns an error + * if they couldn't. + */ +static int process_request(struct scoutfs_net_connection *conn, + struct message_recv *mrecv) +{ + struct super_block *sb = conn->sb; + scoutfs_net_request_t req_func; + + if (mrecv->nh.cmd < SCOUTFS_NET_CMD_UNKNOWN) + req_func = conn->req_funcs[mrecv->nh.cmd]; + else + req_func = NULL; + + if (req_func == NULL) { + scoutfs_inc_counter(sb, net_unknown_request); + return -EINVAL; + } + + return req_func(sb, conn, mrecv->nh.cmd, le64_to_cpu(mrecv->nh.id), + mrecv->nh.data, le16_to_cpu(mrecv->nh.data_len)); +} + +/* + * An incoming response finds the queued request and calls its response + * function. The response function for a given request will only be + * called once. Requests can be canceled while a response is in flight. + * It's not an error to receive a response to a request that no longer + * exists. + */ +static int process_response(struct scoutfs_net_connection *conn, + struct message_recv *mrecv) +{ + struct super_block *sb = conn->sb; + struct message_send *msend; + scoutfs_net_response_t resp_func = NULL; + void *resp_data; + int ret = 0; + + spin_lock(&conn->lock); + + msend = find_request(conn, mrecv->nh.cmd, le64_to_cpu(mrecv->nh.id)); + if (msend) { + resp_func = msend->resp_func; + resp_data = msend->resp_data; + complete_send(conn, msend); + } else { + scoutfs_inc_counter(sb, net_dropped_response); + } + + spin_unlock(&conn->lock); + + if (resp_func) + ret = resp_func(sb, conn, mrecv->nh.data, + le16_to_cpu(mrecv->nh.data_len), + net_err_to_host(mrecv->nh.error), resp_data); + return ret; +} + +/* + * Process an incoming received message in its own concurrent blocking + * work context. + */ +static void scoutfs_net_proc_worker(struct work_struct *work) +{ + struct message_recv *mrecv = container_of(work, struct message_recv, + proc_work); + struct scoutfs_net_connection *conn = mrecv->conn; + struct super_block *sb = conn->sb; + struct net_info *ninf = SCOUTFS_SB(sb)->net_info; + int ret; + + trace_scoutfs_net_proc_work_enter(sb, 0, 0); + + if (nh_is_request(&mrecv->nh)) + ret = process_request(conn, mrecv); + else + ret = process_response(conn, mrecv); + + /* process_one_work explicitly allows freeing work in its func */ + scoutfs_tseq_del(&ninf->msg_tseq_tree, &mrecv->tseq_entry); + kfree(mrecv); + + /* shut down the connection if processing returns fatal errors */ + if (ret) + shutdown_conn(conn); + + trace_scoutfs_net_proc_work_exit(sb, 0, ret); +} + +/* + * Free live responses up to and including the seq by marking them dead + * and moving them to the send queue to be freed. + */ +static int move_acked_responses(struct scoutfs_net_connection *conn, + struct list_head *list, u64 seq) +{ + struct message_send *msend; + struct message_send *tmp; + int ret = 0; + + assert_spin_locked(&conn->lock); + + list_for_each_entry_safe(msend, tmp, list, head) { + if (le64_to_cpu(msend->nh.seq) > seq) + break; + if (!nh_is_response(&msend->nh) || msend->dead) + continue; + + msend->dead = 1; + list_move(&msend->head, &conn->send_queue); + ret = 1; + } + + return ret; +} + +/* acks are processed inline in the recv worker */ +static void free_acked_responses(struct scoutfs_net_connection *conn, u64 seq) +{ + int moved; + + spin_lock(&conn->lock); + + moved = move_acked_responses(conn, &conn->send_queue, seq) + + move_acked_responses(conn, &conn->resend_queue, seq); + + spin_unlock(&conn->lock); + + if (moved) + queue_work(conn->workq, &conn->send_work); +} + +static int recvmsg_full(struct socket *sock, void *buf, unsigned len) +{ + struct msghdr msg; + struct kvec kv; + int ret; + + while (len) { + memset(&msg, 0, sizeof(msg)); + msg.msg_iov = (struct iovec *)&kv; + msg.msg_iovlen = 1; + msg.msg_flags = MSG_NOSIGNAL; + kv.iov_base = buf; + kv.iov_len = len; + + ret = kernel_recvmsg(sock, &msg, &kv, 1, len, msg.msg_flags); + if (ret <= 0) + return -ECONNABORTED; + + len -= ret; + buf += ret; + } + + return 0; +} + +static bool invalid_message(struct scoutfs_net_connection *conn, + struct scoutfs_net_header *nh) +{ + /* seq and id must be non-zero */ + if (nh->seq == 0 || nh->id == 0) + return true; + + /* greeting should negotiate understood protocol */ + if (nh->cmd >= SCOUTFS_NET_CMD_UNKNOWN || + (nh->flags & SCOUTFS_NET_FLAGS_UNKNOWN) || + nh->error >= SCOUTFS_NET_ERR_UNKNOWN) + return true; + + /* payloads have a limit */ + if (le16_to_cpu(nh->data_len) > SCOUTFS_NET_MAX_DATA_LEN) + return true; + + /* only responses can carry errors */ + if (nh_is_request(nh) && nh->error != SCOUTFS_NET_ERR_NONE) + return true; + + if (nh->cmd == SCOUTFS_NET_CMD_GREETING) { + /* each endpoint can only receive one greeting per socket */ + if (test_conn_fl(conn, saw_greeting)) + return true; + + /* servers get greeting requests, clients get responses */ + if (!!conn->listening_conn != !!nh_is_request(nh)) + return true; + } + + return false; +} + +/* + * Always block receiving from the socket. Errors trigger shutting down + * the connection. + */ +static void scoutfs_net_recv_worker(struct work_struct *work) +{ + DEFINE_CONN_FROM_WORK(conn, work, recv_work); + struct super_block *sb = conn->sb; + struct net_info *ninf = SCOUTFS_SB(sb)->net_info; + struct socket *sock = conn->sock; + struct scoutfs_net_header nh; + struct message_recv *mrecv; + unsigned int data_len; + int ret; + + trace_scoutfs_net_recv_work_enter(sb, 0, 0); + + for (;;) { + /* receive the header */ + ret = recvmsg_full(sock, &nh, sizeof(nh)); + if (ret) + break; + + /* receiving an invalid message breaks the connection */ + if (invalid_message(conn, &nh)) { + scoutfs_inc_counter(sb, net_recv_invalid_message); + ret = -EBADMSG; + break; + } + + trace_scoutfs_recv_clock_sync(nh.clock_sync_id); + + data_len = le16_to_cpu(nh.data_len); + + scoutfs_inc_counter(sb, net_recv_messages); + scoutfs_add_counter(sb, net_recv_bytes, nh_bytes(data_len)); + trace_scoutfs_net_recv_message(sb, &conn->sockname, + &conn->peername, &nh); + + /* invalid message checked data len */ + mrecv = kmalloc(offsetof(struct message_recv, + nh.data[data_len]), GFP_NOFS); + if (!mrecv) { + ret = -ENOMEM; + break; + } + + mrecv->conn = conn; + INIT_WORK(&mrecv->proc_work, scoutfs_net_proc_worker); + mrecv->nh = nh; + + /* receive the data payload */ + ret = recvmsg_full(sock, mrecv->nh.data, data_len); + if (ret) { + kfree(mrecv); + break; + } + + if (nh.cmd == SCOUTFS_NET_CMD_GREETING) { + /* greetings are out of band, no seq mechanics */ + set_conn_fl(conn, saw_greeting); + + } else if (le64_to_cpu(nh.seq) <= + atomic64_read(&conn->recv_seq)) { + /* drop any resent duplicated messages */ + scoutfs_inc_counter(sb, net_recv_dropped_duplicate); + kfree(mrecv); + continue; + + } else { + /* record that we've received sender's seq */ + atomic64_set(&conn->recv_seq, le64_to_cpu(nh.seq)); + /* and free our responses that sender has received */ + free_acked_responses(conn, le64_to_cpu(nh.recv_seq)); + } + + scoutfs_tseq_add(&ninf->msg_tseq_tree, &mrecv->tseq_entry); + + /* synchronously process greeting before next recvmsg */ + if (nh.cmd == SCOUTFS_NET_CMD_GREETING) + scoutfs_net_proc_worker(&mrecv->proc_work); + else + queue_work(conn->workq, &mrecv->proc_work); + } + + if (ret) + scoutfs_inc_counter(sb, net_recv_error); + + /* recv stopping always shuts down the connection */ + shutdown_conn(conn); + + trace_scoutfs_net_recv_work_exit(sb, 0, ret); +} + +static int sendmsg_full(struct socket *sock, void *buf, unsigned len) +{ + struct msghdr msg; + struct kvec kv; + int ret; + + while (len) { + memset(&msg, 0, sizeof(msg)); + msg.msg_iov = (struct iovec *)&kv; + msg.msg_iovlen = 1; + msg.msg_flags = MSG_NOSIGNAL; + kv.iov_base = buf; + kv.iov_len = len; + + ret = kernel_sendmsg(sock, &msg, &kv, 1, len); + if (ret <= 0) + return -ECONNABORTED; + + len -= ret; + buf += ret; + } + + return 0; +} + +static void free_msend(struct net_info *ninf, struct message_send *msend) +{ + list_del_init(&msend->head); + scoutfs_tseq_del(&ninf->msg_tseq_tree, &msend->tseq_entry); + kfree(msend); +} + +/* + * Each connection has a single worker that sends queued messages down + * the connection's socket. The work is queued whenever a message is + * put on the send queue. The worker uses blocking sends so that we + * don't have to worry about resuming partial sends or hooking into + * data_ready. Send errors shut down the connection. + * + * The worker is responsible for freeing messages so that other contexts + * don't have to worry about freeing a message while we're blocked + * sending it without the lock held. + * + * We set the current recv_seq on every outgoing frame as it represents + * the current connection state, not the state back when each message + * was first queued. + */ +static void scoutfs_net_send_worker(struct work_struct *work) +{ + DEFINE_CONN_FROM_WORK(conn, work, send_work); + struct super_block *sb = conn->sb; + struct net_info *ninf = SCOUTFS_SB(sb)->net_info; + struct message_send *msend; + int ret = 0; + int len; + + trace_scoutfs_net_send_work_enter(sb, 0, 0); + + spin_lock(&conn->lock); + + while ((msend = list_first_entry_or_null(&conn->send_queue, + struct message_send, head))) { + + if (msend->dead) { + free_msend(ninf, msend); + continue; + } + + if ((msend->nh.cmd == SCOUTFS_NET_CMD_FAREWELL) && + nh_is_response(&msend->nh)) { + set_conn_fl(conn, saw_farewell); + } + + msend->nh.recv_seq = + cpu_to_le64(atomic64_read(&conn->recv_seq)); + + spin_unlock(&conn->lock); + + len = nh_bytes(le16_to_cpu(msend->nh.data_len)); + + scoutfs_inc_counter(sb, net_send_messages); + scoutfs_add_counter(sb, net_send_bytes, len); + trace_scoutfs_net_send_message(sb, &conn->sockname, + &conn->peername, &msend->nh); + + msend->nh.clock_sync_id = scoutfs_clock_sync_id(); + trace_scoutfs_send_clock_sync(msend->nh.clock_sync_id); + + ret = sendmsg_full(conn->sock, &msend->nh, len); + + spin_lock(&conn->lock); + + msend->nh.recv_seq = 0; + + if (ret) + break; + + /* resend if it wasn't freed while we sent */ + if (!msend->dead) + list_move_tail(&msend->head, &conn->resend_queue); + } + + spin_unlock(&conn->lock); + + if (ret) { + scoutfs_inc_counter(sb, net_send_error); + shutdown_conn(conn); + } + + trace_scoutfs_net_send_work_exit(sb, 0, ret); +} + +/* + * Listening conns try to destroy accepted conns. Workqueues model + * flushing work as acquiring a workqueue class lock so it thinks that + * this is a deadlock because it doesn't know about our hierarchy of + * workqueues. The workqueue lockdep_map is private so we can't set a + * subclass to differentiate between listening and accepted conn + * workqueues. Instead we queue final conn destruction off to a longer + * lived specific workqueue that has a different class. + */ +static void scoutfs_net_destroy_worker(struct work_struct *work) +{ + DEFINE_CONN_FROM_WORK(conn, work, destroy_work); + struct super_block *sb = conn->sb; + struct net_info *ninf = SCOUTFS_SB(sb)->net_info; + struct scoutfs_net_connection *listener; + struct message_send *msend; + struct message_send *tmp; + + trace_scoutfs_net_destroy_work_enter(sb, 0, 0); + trace_scoutfs_conn_destroy_start(conn); + + WARN_ON_ONCE(conn->sock != NULL); + WARN_ON_ONCE(!list_empty(&conn->accepted_list)); + + /* tell callers that accepted connection finally done */ + if (conn->listening_conn && conn->notify_down) + conn->notify_down(sb, conn, conn->info, conn->rid); + + /* free all messages, refactor and complete for forced unmount? */ + list_splice_init(&conn->resend_queue, &conn->send_queue); + list_for_each_entry_safe(msend, tmp, &conn->send_queue, head) { + free_msend(ninf, msend); + } + + /* accepted sockets are removed from their listener's list */ + if (conn->listening_conn) { + listener = conn->listening_conn; + + spin_lock(&listener->lock); + list_del_init(&conn->accepted_head); + if (list_empty(&listener->accepted_list)) + wake_up(&listener->waitq); + spin_unlock(&listener->lock); + } + + destroy_workqueue(conn->workq); + scoutfs_tseq_del(&ninf->conn_tseq_tree, &conn->tseq_entry); + kfree(conn->info); + trace_scoutfs_conn_destroy_free(conn); + kfree(conn); + + trace_scoutfs_net_destroy_work_exit(sb, 0, 0); +} + +static void destroy_conn(struct scoutfs_net_connection *conn) +{ + struct net_info *ninf = SCOUTFS_SB(conn->sb)->net_info; + + queue_work(ninf->destroy_workq, &conn->destroy_work); +} + +/* + * Have a pretty aggressive keepalive timeout of around 10 seconds. The + * TCP keepalives are being processed out of task context so they should + * be responsive even when mounts are under load. + */ +#define KEEPCNT 3 +#define KEEPIDLE 7 +#define KEEPINTVL 1 +static int sock_opts_and_names(struct scoutfs_net_connection *conn, + struct socket *sock) +{ + struct timeval tv; + int addrlen; + int optval; + int ret; + + /* but use a keepalive timeout instead of send timeout */ + tv.tv_sec = 0; + tv.tv_usec = 0; + ret = kernel_setsockopt(sock, SOL_SOCKET, SO_SNDTIMEO, + (char *)&tv, sizeof(tv)); + if (ret) + goto out; + + optval = KEEPCNT; + ret = kernel_setsockopt(sock, SOL_TCP, TCP_KEEPCNT, + (char *)&optval, sizeof(optval)); + if (ret) + goto out; + + optval = KEEPIDLE; + ret = kernel_setsockopt(sock, SOL_TCP, TCP_KEEPIDLE, + (char *)&optval, sizeof(optval)); + if (ret) + goto out; + + optval = KEEPINTVL; + ret = kernel_setsockopt(sock, SOL_TCP, TCP_KEEPINTVL, + (char *)&optval, sizeof(optval)); + if (ret) + goto out; + + optval = 1; + ret = kernel_setsockopt(sock, SOL_SOCKET, SO_KEEPALIVE, + (char *)&optval, sizeof(optval)); + if (ret) + goto out; + + optval = 1; + ret = kernel_setsockopt(sock, SOL_TCP, TCP_NODELAY, + (char *)&optval, sizeof(optval)); + if (ret) + goto out; + + addrlen = sizeof(struct sockaddr_in); + ret = kernel_getsockname(sock, (struct sockaddr *)&conn->sockname, + &addrlen); + if (ret == 0 && addrlen != sizeof(struct sockaddr_in)) + ret = -EAFNOSUPPORT; + if (ret) + goto out; + + addrlen = sizeof(struct sockaddr_in); + ret = kernel_getpeername(sock, (struct sockaddr *)&conn->peername, + &addrlen); + if (ret == 0 && addrlen != sizeof(struct sockaddr_in)) + ret = -EAFNOSUPPORT; + if (ret) + goto out; +out: + return ret; +} + +/* + * Each bound and listening connection has long running work that blocks + * accepting new connections. The listening socket has been setup by + * the time this is queued. + * + * Any errors on the listening sock tear down all the connections that + * were accepted. + */ +static void scoutfs_net_listen_worker(struct work_struct *work) +{ + DEFINE_CONN_FROM_WORK(conn, work, listen_work); + struct super_block *sb = conn->sb; + struct scoutfs_net_connection *acc_conn; + DECLARE_WAIT_QUEUE_HEAD(waitq); + struct socket *acc_sock; + LIST_HEAD(conn_list); + int ret; + + trace_scoutfs_net_listen_work_enter(sb, 0, 0); + + for (;;) { + ret = kernel_accept(conn->sock, &acc_sock, 0); + if (ret < 0) + break; + + /* inherit accepted request funcs from listening conn */ + acc_conn = scoutfs_net_alloc_conn(sb, conn->notify_up, + conn->notify_down, + conn->info_size, + conn->req_funcs, "accepted"); + if (!acc_conn) { + sock_release(acc_sock); + ret = -ENOMEM; + continue; + } + + ret = sock_opts_and_names(acc_conn, acc_sock); + if (ret) { + sock_release(acc_sock); + destroy_conn(acc_conn); + continue; + } + + scoutfs_info(sb, "server accepted "SIN_FMT" -> "SIN_FMT, + SIN_ARG(&acc_conn->sockname), + SIN_ARG(&acc_conn->peername)); + + /* acc_conn isn't visible, conn unlock orders stores */ + spin_lock(&conn->lock); + + acc_conn->sock = acc_sock; + acc_conn->listening_conn = conn; + set_conn_fl(acc_conn, established); + list_add_tail(&acc_conn->accepted_head, &conn->accepted_list); + + trace_scoutfs_conn_accept(acc_conn); + + spin_unlock(&conn->lock); + + queue_work(acc_conn->workq, &acc_conn->recv_work); + } + + /* listening stopping shuts down connection */ + shutdown_conn(conn); + + trace_scoutfs_net_listen_work_exit(sb, 0, ret); +} + +/* + * Try once to connect to the caller's address. This is racing with + * shutdown if the caller frees the connection while we're connecting. + * Shutdown will wait for our executing work to finish. + */ +static void scoutfs_net_connect_worker(struct work_struct *work) +{ + DEFINE_CONN_FROM_WORK(conn, work, connect_work); + struct super_block *sb = conn->sb; + struct socket *sock; + struct timeval tv; + int ret; + + trace_scoutfs_net_connect_work_enter(sb, 0, 0); + + ret = sock_create_kern(AF_INET, SOCK_STREAM, IPPROTO_TCP, &sock); + if (ret) + goto out; + + /* caller specified connect timeout */ + tv.tv_sec = conn->connect_timeout_ms / MSEC_PER_SEC; + tv.tv_usec = (conn->connect_timeout_ms % MSEC_PER_SEC) * USEC_PER_MSEC; + ret = kernel_setsockopt(sock, SOL_SOCKET, SO_SNDTIMEO, + (char *)&tv, sizeof(tv)); + if (ret) { + sock_release(sock); + goto out; + } + + /* shutdown now owns sock, can break blocking connect */ + spin_lock(&conn->lock); + conn->sock = sock; + spin_unlock(&conn->lock); + + trace_scoutfs_conn_connect_start(conn); + + ret = kernel_connect(sock, (struct sockaddr *)&conn->connect_sin, + sizeof(struct sockaddr_in), 0); + if (ret) + goto out; + + ret = sock_opts_and_names(conn, sock); + if (ret) + goto out; + + scoutfs_info(sb, "client connected "SIN_FMT" -> "SIN_FMT, + SIN_ARG(&conn->sockname), + SIN_ARG(&conn->peername)); + + spin_lock(&conn->lock); + + /* clear greeting state for next negotiation */ + clear_conn_fl(conn, valid_greeting); + set_conn_fl(conn, established); + wake_up(&conn->waitq); + + trace_scoutfs_conn_connect_complete(conn); + + spin_unlock(&conn->lock); + + queue_work(conn->workq, &conn->recv_work); +out: + if (ret) + shutdown_conn(conn); + + trace_scoutfs_net_connect_work_exit(sb, 0, ret); +} + +static bool empty_accepted_list(struct scoutfs_net_connection *conn) +{ + bool empty; + + spin_lock(&conn->lock); + empty = list_empty(&conn->accepted_list); + spin_unlock(&conn->lock); + + return empty; +} + +/* + * Safely shut down an active connection. This can be triggered by + * errors in workers or by an external call to free the connection. The + * shutting down flag ensures that this only executes once for each live + * socket. + */ +static void scoutfs_net_shutdown_worker(struct work_struct *work) +{ + DEFINE_CONN_FROM_WORK(conn, work, shutdown_work); + struct super_block *sb = conn->sb; + struct net_info *ninf = SCOUTFS_SB(sb)->net_info; + struct scoutfs_net_connection *listener; + struct scoutfs_net_connection *acc_conn; + struct message_send *msend; + struct message_send *tmp; + unsigned long delay; + + trace_scoutfs_net_shutdown_work_enter(sb, 0, 0); + trace_scoutfs_conn_shutdown_start(conn); + + /* connected and accepted conns print a message */ + if (conn->peername.sin_port != 0) + scoutfs_info(sb, "%s "SIN_FMT" -> "SIN_FMT, + conn->listening_conn ? "server closing" : + "client disconnected", + SIN_ARG(&conn->sockname), + SIN_ARG(&conn->peername)); + + /* ensure that sockets return errors, wakes blocked socket work */ + if (conn->sock) + kernel_sock_shutdown(conn->sock, SHUT_RDWR); + + /* wait for socket and proc work to finish, includes chained work */ + drain_workqueue(conn->workq); + + /* tear down the sock now that all work is done */ + if (conn->sock) { + sock_release(conn->sock); + conn->sock = NULL; + } + + memset(&conn->peername, 0, sizeof(conn->peername)); + + /* listening connections shut down all the connections they accepted */ + spin_lock_nested(&conn->lock, CONN_LOCK_LISTENER); + list_for_each_entry(acc_conn, &conn->accepted_list, accepted_head) { + spin_lock_nested(&acc_conn->lock, CONN_LOCK_ACCEPTED); + shutdown_conn_locked(acc_conn); + spin_unlock(&acc_conn->lock); + } + spin_unlock(&conn->lock); + + /* free any conns waiting for reconnection */ + cancel_delayed_work_sync(&conn->reconn_free_dwork); + queue_delayed_work(conn->workq, &conn->reconn_free_dwork, 0); + /* relies on delay 0 scheduling immediately so no timer to cancel */ + flush_delayed_work(&conn->reconn_free_dwork); + + /* and wait for accepted conn shutdown work to finish */ + wait_event(conn->waitq, empty_accepted_list(conn)); + + spin_lock(&conn->lock); + + /* greetings aren't resent across sockets */ + list_splice_tail_init(&conn->send_queue, &conn->resend_queue); + list_for_each_entry_safe(msend, tmp, &conn->resend_queue, head) { + if (msend->nh.cmd == SCOUTFS_NET_CMD_GREETING) + free_msend(ninf, msend); + } + + clear_conn_fl(conn, saw_greeting); + + /* signal connect failure */ + memset(&conn->connect_sin, 0, sizeof(conn->connect_sin)); + wake_up(&conn->waitq); + + /* resolve racing with listener shutdown with locked shutting_down */ + if (conn->listening_conn && + (test_conn_fl(conn->listening_conn, shutting_down) || + test_conn_fl(conn, saw_farewell))) { + + /* free accepted sockets after farewell or listener shutdown */ + spin_unlock(&conn->lock); + destroy_conn(conn); + + } else { + + if (conn->listening_conn) { + /* server accepted sockets wait for reconnect */ + listener = conn->listening_conn; + delay = msecs_to_jiffies(CLIENT_RECONNECT_TIMEOUT_MS); + set_conn_fl(conn, reconn_wait); + conn->reconn_deadline = jiffies + delay; + queue_delayed_work(listener->workq, + &listener->reconn_free_dwork, delay); + } else { + /* clients and listeners can retry */ + clear_conn_fl(conn, shutting_down); + if (conn->notify_down) + conn->notify_down(sb, conn, conn->info, + conn->rid); + } + + trace_scoutfs_conn_shutdown_complete(conn); + spin_unlock(&conn->lock); + } + + trace_scoutfs_net_shutdown_work_exit(sb, 0, 0); +} + +/* + * Free any connections that have been shutdown for too long without the + * client reconnecting. This runs in work on the listening connection. + * It's racing with connection attempts searching for shutdown + * connections to steal state from. Shutdown cancels the work and waits + * for it to finish. + * + * Connections are currently freed without the lock held so this walks + * the entire list every time it frees a connection. This is irritating + * but timed out connections are rare and client counts are relatively + * low given a cpu's ability to burn through the list. + */ +static void scoutfs_net_reconn_free_worker(struct work_struct *work) +{ + DEFINE_CONN_FROM_WORK(conn, work, reconn_free_dwork.work); + struct super_block *sb = conn->sb; + struct scoutfs_net_connection *acc; + unsigned long now = jiffies; + unsigned long deadline = 0; + bool requeue = false; + + trace_scoutfs_net_reconn_free_work_enter(sb, 0, 0); + +restart: + spin_lock(&conn->lock); + list_for_each_entry(acc, &conn->accepted_list, accepted_head) { + + if (test_conn_fl(acc, reconn_wait) && + !test_conn_fl(acc, reconn_freeing) && + (test_conn_fl(conn, shutting_down) || + time_after_eq(now, acc->reconn_deadline))) { + set_conn_fl(acc, reconn_freeing); + spin_unlock(&conn->lock); + if (!test_conn_fl(conn, shutting_down)) + scoutfs_info(sb, "client timed out "SIN_FMT" -> "SIN_FMT", can not reconnect", + SIN_ARG(&acc->sockname), + SIN_ARG(&acc->peername)); + destroy_conn(acc); + goto restart; + } + + /* calc delay of next work, can drift a bit */ + if (test_conn_fl(acc, reconn_wait) && + !test_conn_fl(acc, reconn_freeing) && + (!requeue || time_before(now, deadline))) { + requeue = true; + deadline = acc->reconn_deadline; + } + } + spin_unlock(&conn->lock); + + if (requeue) + queue_delayed_work(conn->workq, &conn->reconn_free_dwork, + deadline - now); + + trace_scoutfs_net_reconn_free_work_exit(sb, 0, 0); +} + +/* + * Accepted connections inherit the callbacks from their listening + * connection. + * + * notify_up is called once a valid greeting is received. rid is + * non-zero on accepted sockets once they've seen a valid greeting. + * Connected and listening connections have a rid of 0. + * + * notify_down is always called as connections are shut down. It can be + * called without notify_up ever being called. The rid is only + * non-zero for accepted connections. + */ +struct scoutfs_net_connection * +scoutfs_net_alloc_conn(struct super_block *sb, + scoutfs_net_notify_t notify_up, + scoutfs_net_notify_t notify_down, size_t info_size, + scoutfs_net_request_t *req_funcs, char *name_suffix) +{ + struct net_info *ninf = SCOUTFS_SB(sb)->net_info; + struct scoutfs_net_connection *conn; + + conn = kzalloc(sizeof(struct scoutfs_net_connection), GFP_NOFS); + if (!conn) + return NULL; + + conn->info = kzalloc(info_size, GFP_NOFS); + if (!conn->info) { + kfree(conn); + return NULL; + } + + conn->workq = alloc_workqueue("scoutfs_net_%s", + WQ_UNBOUND | WQ_NON_REENTRANT, 0, + name_suffix); + if (!conn->workq) { + kfree(conn->info); + kfree(conn); + return NULL; + } + + conn->sb = sb; + conn->notify_up = notify_up; + conn->notify_down = notify_down; + conn->info_size = info_size; + conn->req_funcs = req_funcs; + spin_lock_init(&conn->lock); + init_waitqueue_head(&conn->waitq); + conn->sockname.sin_family = AF_INET; + conn->peername.sin_family = AF_INET; + INIT_LIST_HEAD(&conn->accepted_head); + INIT_LIST_HEAD(&conn->accepted_list); + conn->next_send_seq = 1; + conn->next_send_id = 1; + atomic64_set(&conn->recv_seq, 0); + INIT_LIST_HEAD(&conn->send_queue); + INIT_LIST_HEAD(&conn->resend_queue); + INIT_WORK(&conn->listen_work, scoutfs_net_listen_worker); + INIT_WORK(&conn->connect_work, scoutfs_net_connect_worker); + INIT_WORK(&conn->send_work, scoutfs_net_send_worker); + INIT_WORK(&conn->recv_work, scoutfs_net_recv_worker); + INIT_WORK(&conn->shutdown_work, scoutfs_net_shutdown_worker); + INIT_WORK(&conn->destroy_work, scoutfs_net_destroy_worker); + INIT_DELAYED_WORK(&conn->reconn_free_dwork, + scoutfs_net_reconn_free_worker); + + scoutfs_tseq_add(&ninf->conn_tseq_tree, &conn->tseq_entry); + trace_scoutfs_conn_alloc(conn); + + return conn; +} + +/* + * Give the caller the client rid of the connection. This used by rare + * server processing callers who want to send async responses after + * request processing has returned. We didn't want the churn of + * providing the requesting rid to all the request handlers, but we + * probably should. + */ +u64 scoutfs_net_client_rid(struct scoutfs_net_connection *conn) +{ + return conn->rid; +} + +/* + * Shutdown the connection. Once this returns no network traffic + * or work will be executing. The caller can then connect or bind and + * listen again. Additional shutdown calls will already find it shutdown. + */ +void scoutfs_net_shutdown(struct super_block *sb, + struct scoutfs_net_connection *conn) +{ + shutdown_conn(conn); + flush_work(&conn->shutdown_work); + flush_work(&conn->destroy_work); +} + +/* + * Destroy the connection after the shutdown work has stopped all concurrent + * processing on the connection. + */ +void scoutfs_net_free_conn(struct super_block *sb, + struct scoutfs_net_connection *conn) +{ + if (conn) { + scoutfs_net_shutdown(sb, conn); + destroy_conn(conn); + } +} + +/* + * Associate a bound socket with the caller's connection. We call bind + * and listen to assign the listening address and give it to the caller. + * + * If this returns success then the caller has to call either listen or + * free_conn. + */ +int scoutfs_net_bind(struct super_block *sb, + struct scoutfs_net_connection *conn, + struct sockaddr_in *sin) +{ + struct socket *sock = NULL; + int addrlen; + int optval; + int ret; + + /* caller state machine shouldn't let this happen */ + if (WARN_ON_ONCE(conn->sock)) + return -EINVAL; + + ret = sock_create_kern(AF_INET, SOCK_STREAM, IPPROTO_TCP, &sock); + if (ret) + goto out; + + optval = 1; + ret = kernel_setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, + (char *)&optval, sizeof(optval)); + if (ret) + goto out; + + addrlen = sizeof(struct sockaddr_in); + ret = kernel_bind(sock, (struct sockaddr *)sin, addrlen); + if (ret) + goto out; + + ret = kernel_listen(sock, 255); + if (ret) + goto out; + + addrlen = sizeof(struct sockaddr_in); + ret = kernel_getsockname(sock, (struct sockaddr *)&conn->sockname, + &addrlen); + if (ret == 0 && addrlen != sizeof(struct sockaddr_in)) + ret = -EAFNOSUPPORT; + if (ret) + goto out; + + conn->sock = sock; + *sin = conn->sockname; + ret = 0; +out: + if (ret < 0 && sock) + sock_release(sock); + return ret; +} + +/* + * Kick off blocking background work to accept connections from the + * connection's listening socket that was created with a previous bind + * call. + * + * The callback notify_down will be called once the listening socket is + * shut down either by errors or the caller freeing the conn. + */ +void scoutfs_net_listen(struct super_block *sb, + struct scoutfs_net_connection *conn) +{ + queue_work(conn->workq, &conn->listen_work); +} + +/* + * Return once a connection attempt has completed either successfully + * or in error. + */ +static bool connect_result(struct scoutfs_net_connection *conn, int *error) +{ + bool done = false; + + spin_lock(&conn->lock); + if (test_conn_fl(conn, established)) { + done = true; + *error = 0; + } else if (test_conn_fl(conn, shutting_down) || + conn->connect_sin.sin_family == 0) { + done = true; + *error = -ESHUTDOWN; + } + trace_scoutfs_conn_connect_result(conn); + spin_unlock(&conn->lock); + + return done; +} + +/* + * Connect to the given address. An error is returned if the socket was + * not connected before the given timeout. The connection isn't fully + * active until the connecting caller starts greeting negotiation by + * sending the initial greeting request. + * + * The conn notify_down callback can be called as the connection is + * shutdown before this returns. + */ +int scoutfs_net_connect(struct super_block *sb, + struct scoutfs_net_connection *conn, + struct sockaddr_in *sin, unsigned long timeout_ms) +{ + int error = 0; + int ret; + + spin_lock(&conn->lock); + conn->connect_sin = *sin; + conn->connect_timeout_ms = timeout_ms; + spin_unlock(&conn->lock); + + queue_work(conn->workq, &conn->connect_work); + + ret = wait_event_interruptible(conn->waitq, + connect_result(conn, &error)); + return ret ?: error; +} + +static void set_valid_greeting(struct scoutfs_net_connection *conn) +{ + assert_spin_locked(&conn->lock); + + /* recv should have dropped invalid duplicate greeting messages */ + BUG_ON(test_conn_fl(conn, valid_greeting)); + + set_conn_fl(conn, valid_greeting); + list_splice_tail_init(&conn->resend_queue, &conn->send_queue); + queue_work(conn->workq, &conn->send_work); +} + +/* + * The client has received a valid greeting from the server. Send + * can proceed and we might need to reset our recv state if we reconnected + * to a new server. + */ +void scoutfs_net_client_greeting(struct super_block *sb, + struct scoutfs_net_connection *conn, + bool new_server) +{ + struct net_info *ninf = SCOUTFS_SB(sb)->net_info; + struct message_send *msend; + struct message_send *tmp; + + /* only called on client connections :/ */ + BUG_ON(conn->listening_conn); + + spin_lock(&conn->lock); + + if (new_server) { + atomic64_set(&conn->recv_seq, 0); + list_for_each_entry_safe(msend, tmp, &conn->resend_queue, head){ + if (nh_is_response(&msend->nh)) + free_msend(ninf, msend); + } + } + + set_valid_greeting(conn); + + spin_unlock(&conn->lock); + + /* client up/down drives reconnect */ + if (conn->notify_up) + conn->notify_up(sb, conn, conn->info, 0); +} + +/* + * The calling server has received a valid greeting from a client. If + * the client is reconnecting to us then we need to find its old + * connection that held its state and transfer it to this connection + * (connection and socket life cycles make this easier than migrating + * the socket between the connections). + * + * The previous connection that holds the client's state might still be + * in active use depending on network failure and work processing races. + * We shut it down before migrating its message state. We can be + * processing greetings from multiple reconnecting sockets that are all + * referring to the same original connection. We use the increasing + * greeting id to have the most recent connection attempt win. + * + * A node can be reconnecting to us for the first time. It will notice + * the new server term and take steps to recover. + * + * A client can be reconnecting to us after we've destroyed their state. + * This is fatal for the client if they just took too long to reconnect. + * But this can also happen if something disconnects the socket after + * we've sent a farewell response before the client received it. In + * this case we let the client reconnect so we can resend the farewell + * response and they can disconnect cleanly. + * + * At this point our connection is idle except for send submissions and + * shutdown being queued. Once we shut down a We completely own a We + * have exclusive access to a previous conn once its shutdown and we set + * _freeing. + */ +void scoutfs_net_server_greeting(struct super_block *sb, + struct scoutfs_net_connection *conn, + u64 rid, u64 greeting_id, + bool reconnecting, bool first_contact, + bool farewell) +{ + struct scoutfs_net_connection *listener; + struct scoutfs_net_connection *reconn; + struct scoutfs_net_connection *acc; + + /* only called on accepted server connections :/ */ + BUG_ON(!conn->listening_conn); + + /* see if we have a previous conn for the client's sent rid */ + reconn = NULL; + if (reconnecting) { + listener = conn->listening_conn; +restart: + spin_lock_nested(&listener->lock, CONN_LOCK_LISTENER); + list_for_each_entry(acc, &listener->accepted_list, + accepted_head) { + if (acc->rid != rid || + acc->greeting_id >= greeting_id || + test_conn_fl(acc, reconn_freeing)) + continue; + + if (!test_conn_fl(acc, reconn_wait)) { + spin_lock_nested(&acc->lock, + CONN_LOCK_ACCEPTED); + shutdown_conn_locked(acc); + spin_unlock(&acc->lock); + spin_unlock(&listener->lock); + msleep(10); /* XXX might be freed :/ */ + goto restart; + } + + reconn = acc; + set_conn_fl(reconn, reconn_freeing); + break; + } + spin_unlock(&listener->lock); + } + + /* drop a connection if we can't find its necessary old conn */ + if (reconnecting && !reconn && !first_contact && !farewell) { + shutdown_conn(conn); + return; + } + + /* migrate state from previous conn for this reconnecting rid */ + if (reconn) { + spin_lock(&conn->lock); + + assign_conn_fl(conn, reconn, saw_farewell); + conn->next_send_seq = reconn->next_send_seq; + conn->next_send_id = reconn->next_send_id; + atomic64_set(&conn->recv_seq, atomic64_read(&reconn->recv_seq)); + + /* greeting response/ack will be on conn send queue */ + BUG_ON(!list_empty(&reconn->send_queue)); + BUG_ON(!list_empty(&conn->resend_queue)); + list_splice_init(&reconn->resend_queue, &conn->resend_queue); + + /* new conn info is unused, swap, old won't call down */ + swap(conn->info, reconn->info); + reconn->notify_down = NULL; + + trace_scoutfs_conn_reconn_migrate(conn); + spin_unlock(&conn->lock); + + /* we set _freeing */ + destroy_conn(reconn); + } + + spin_lock(&conn->lock); + + conn->rid = rid; + conn->greeting_id = greeting_id; + set_valid_greeting(conn); + + spin_unlock(&conn->lock); + + /* only call notify_up the first time we see the rid */ + if (conn->notify_up && first_contact) + conn->notify_up(sb, conn, conn->info, rid); +} + +/* + * Submit a request down the connection. It's up to the caller to + * ensure that the conn is allocated. Sends submitted when the + * connection isn't established will be resent in order the next time + * it's established. + */ +int scoutfs_net_submit_request(struct super_block *sb, + struct scoutfs_net_connection *conn, + u8 cmd, void *arg, u16 arg_len, + scoutfs_net_response_t resp_func, + void *resp_data, u64 *id_ret) +{ + return submit_send(sb, conn, 0, cmd, 0, 0, 0, arg, arg_len, + resp_func, resp_data, id_ret); +} + +/* + * Send a request to a specific rid that was accepted by this listening + * connection. + */ +int scoutfs_net_submit_request_node(struct super_block *sb, + struct scoutfs_net_connection *conn, + u64 rid, u8 cmd, + void *arg, u16 arg_len, + scoutfs_net_response_t resp_func, + void *resp_data, u64 *id_ret) +{ + return submit_send(sb, conn, rid, cmd, 0, 0, 0, arg, arg_len, + resp_func, resp_data, id_ret); +} + +/* + * Send a response. Responses don't get callbacks and use the request's + * id so caller's don't need to get an id in return. + * + * An error is returned if the response could not be sent. + */ +int scoutfs_net_response(struct super_block *sb, + struct scoutfs_net_connection *conn, + u8 cmd, u64 id, int error, void *resp, u16 resp_len) +{ + if (error) { + resp = NULL; + resp_len = 0; + } + + return submit_send(sb, conn, 0, cmd, SCOUTFS_NET_FLAG_RESPONSE, id, + net_err_from_host(sb, error), resp, resp_len, + NULL, NULL, NULL); +} + +int scoutfs_net_response_node(struct super_block *sb, + struct scoutfs_net_connection *conn, + u64 rid, u8 cmd, u64 id, int error, + void *resp, u16 resp_len) +{ + if (error) { + resp = NULL; + resp_len = 0; + } + + return submit_send(sb, conn, rid, cmd, SCOUTFS_NET_FLAG_RESPONSE, + id, net_err_from_host(sb, error), resp, resp_len, + NULL, NULL, NULL); +} + +/* + * The response function that was submitted with the request is not + * called if the request is canceled here. + */ +void scoutfs_net_cancel_request(struct super_block *sb, + struct scoutfs_net_connection *conn, + u8 cmd, u64 id) +{ + struct message_send *msend; + + spin_lock(&conn->lock); + msend = find_request(conn, cmd, id); + if (msend) + complete_send(conn, msend); + spin_unlock(&conn->lock); +} + +struct sync_request_completion { + struct completion comp; + void *resp; + unsigned int resp_len; + int error; +}; + +static int sync_response(struct super_block *sb, + struct scoutfs_net_connection *conn, + void *resp, unsigned int resp_len, + int error, void *data) +{ + struct sync_request_completion *sreq = data; + + if (error == 0 && resp_len != sreq->resp_len) + error = -EMSGSIZE; + + if (error) + sreq->error = error; + else if (resp_len) + memcpy(sreq->resp, resp, resp_len); + + complete(&sreq->comp); + + return 0; +} + +/* + * Send a request and wait for a response to be copied into the given + * buffer. Errors returned can come from the remote request processing + * or local failure to send. + * + * The wait for the response is interruptible and can return + * -ERESTARTSYS if it is interrupted. + * + * -EOVERFLOW is returned if the response message's data_length doesn't + * match the caller's resp_len buffer. + */ +int scoutfs_net_sync_request(struct super_block *sb, + struct scoutfs_net_connection *conn, + u8 cmd, void *arg, unsigned arg_len, + void *resp, size_t resp_len) +{ + struct sync_request_completion sreq; + int ret; + u64 id; + + init_completion(&sreq.comp); + sreq.resp = resp; + sreq.resp_len = resp_len; + sreq.error = 0; + + ret = scoutfs_net_submit_request(sb, conn, cmd, arg, arg_len, + sync_response, &sreq, &id); + + ret = wait_for_completion_interruptible(&sreq.comp); + if (ret == -ERESTARTSYS) + scoutfs_net_cancel_request(sb, conn, cmd, id); + else + ret = sreq.error; + + return ret; +} + +static void net_tseq_show_conn(struct seq_file *m, + struct scoutfs_tseq_entry *ent) +{ + struct scoutfs_net_connection *conn = + container_of(ent, struct scoutfs_net_connection, tseq_entry); + + seq_printf(m, "name "SIN_FMT" peer "SIN_FMT" rid %016llx greeting_id %llu vg %u est %u sd %u sg %u sf %u rw %u rf %u cto_ms rdl_j %lu %lu nss %llu rs %llu nsi %llu\n", + SIN_ARG(&conn->sockname), SIN_ARG(&conn->peername), + conn->rid, conn->greeting_id, + test_conn_fl(conn, valid_greeting), + test_conn_fl(conn, established), + test_conn_fl(conn, shutting_down), + test_conn_fl(conn, saw_greeting), + test_conn_fl(conn, saw_farewell), + test_conn_fl(conn, reconn_wait), + test_conn_fl(conn, reconn_freeing), + conn->connect_timeout_ms, conn->reconn_deadline, + conn->next_send_seq, (u64)atomic64_read(&conn->recv_seq), + conn->next_send_id); +} + +/* + * How's this for sneaky?! We line up the structs so that the entries + * and function pointers are at the same offsets. recv's function + * pointer value is known and can't be found in send's. + */ +static bool tseq_entry_is_recv(struct scoutfs_tseq_entry *ent) +{ + struct message_recv *mrecv = + container_of(ent, struct message_recv, tseq_entry); + + BUILD_BUG_ON(offsetof(struct message_recv, tseq_entry) != + offsetof(struct message_send, tseq_entry)); + BUILD_BUG_ON(offsetof(struct message_recv, proc_work.func) != + offsetof(struct message_send, resp_func)); + + return mrecv->proc_work.func == scoutfs_net_proc_worker; +} + +static void net_tseq_show_msg(struct seq_file *m, + struct scoutfs_tseq_entry *ent) +{ + struct message_send *msend; + struct message_recv *mrecv; + + if (tseq_entry_is_recv(ent)) { + mrecv = container_of(ent, struct message_recv, tseq_entry); + + seq_printf(m, "recv "SNH_FMT"\n", SNH_ARG(&mrecv->nh)); + } else { + msend = container_of(ent, struct message_send, tseq_entry); + + seq_printf(m, "send "SNH_FMT"\n", SNH_ARG(&msend->nh)); + } +} + +int scoutfs_net_setup(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct net_info *ninf; + int ret; + + /* fail the build if host errnos don't fit in the u8 mapping arrays */ +#undef EXPAND_NET_ERRNO +#define EXPAND_NET_ERRNO(which) BUILD_BUG_ON(which >= U8_MAX); + EXPAND_EACH_NET_ERRNO + + ninf = kzalloc(sizeof(struct net_info), GFP_KERNEL); + if (!ninf) { + ret = -ENOMEM; + goto out; + } + sbi->net_info = ninf; + + scoutfs_tseq_tree_init(&ninf->conn_tseq_tree, net_tseq_show_conn); + scoutfs_tseq_tree_init(&ninf->msg_tseq_tree, net_tseq_show_msg); + + ninf->shutdown_workq = alloc_workqueue("scoutfs_net_shutdown", + WQ_UNBOUND | WQ_NON_REENTRANT, + 0); + ninf->destroy_workq = alloc_workqueue("scoutfs_net_destroy", + WQ_UNBOUND | WQ_NON_REENTRANT, + 0); + if (!ninf->shutdown_workq || !ninf->destroy_workq) { + ret = -ENOMEM; + goto out; + } + + ninf->conn_tseq_dentry = scoutfs_tseq_create("connections", + sbi->debug_root, + &ninf->conn_tseq_tree); + if (!ninf->conn_tseq_dentry) { + ret = -ENOMEM; + goto out; + } + + ninf->msg_tseq_dentry = scoutfs_tseq_create("messages", + sbi->debug_root, + &ninf->msg_tseq_tree); + if (!ninf->msg_tseq_dentry) { + ret = -ENOMEM; + goto out; + } + + ret = 0; +out: + if (ret) + scoutfs_net_destroy(sb); + return ret; +} + +void scoutfs_net_destroy(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct net_info *ninf = SCOUTFS_SB(sb)->net_info; + + if (ninf) { + if (ninf->shutdown_workq) + destroy_workqueue(ninf->shutdown_workq); + if (ninf->destroy_workq) + destroy_workqueue(ninf->destroy_workq); + debugfs_remove(ninf->conn_tseq_dentry); + debugfs_remove(ninf->msg_tseq_dentry); + kfree(ninf); + sbi->net_info = NULL; + } +} diff --git a/kmod/src/net.h b/kmod/src/net.h new file mode 100644 index 00000000..05e9c3be --- /dev/null +++ b/kmod/src/net.h @@ -0,0 +1,165 @@ +#ifndef _SCOUTFS_NET_H_ +#define _SCOUTFS_NET_H_ + +#include +#include "endian_swap.h" +#include "tseq.h" + +struct scoutfs_net_connection; + +/* These are called in their own blocking context */ +typedef int (*scoutfs_net_request_t)(struct super_block *sb, + struct scoutfs_net_connection *conn, + u8 cmd, u64 id, void *arg, u16 arg_len); + +/* These are called in their own blocking context */ +typedef int (*scoutfs_net_response_t)(struct super_block *sb, + struct scoutfs_net_connection *conn, + void *resp, unsigned int resp_len, + int error, void *data); + +typedef void (*scoutfs_net_notify_t)(struct super_block *sb, + struct scoutfs_net_connection *conn, + void *info, u64 rid); + +/* + * The conn is only here so that tracing can get at its fields without + * having trace functions with a trillion arguments. Tracing requires + * duplicating the arguments for every event, no thanks. + */ + +struct scoutfs_net_connection { + struct super_block *sb; + scoutfs_net_notify_t notify_up; + scoutfs_net_notify_t notify_down; + size_t info_size; + scoutfs_net_request_t *req_funcs; + + spinlock_t lock; + wait_queue_head_t waitq; + + unsigned long flags; /* CONN_FL_* bitmask */ + unsigned long reconn_deadline; + + struct sockaddr_in connect_sin; + unsigned long connect_timeout_ms; + + struct socket *sock; + u64 rid; + u64 greeting_id; + struct sockaddr_in sockname; + struct sockaddr_in peername; + + struct list_head accepted_head; + struct scoutfs_net_connection *listening_conn; + struct list_head accepted_list; + + u64 next_send_seq; + u64 next_send_id; + struct list_head send_queue; + struct list_head resend_queue; + + atomic64_t recv_seq; + + struct workqueue_struct *workq; + struct work_struct listen_work; + struct work_struct connect_work; + struct work_struct send_work; + struct work_struct recv_work; + struct work_struct shutdown_work; + struct work_struct destroy_work; + struct delayed_work reconn_free_dwork; + /* message_recv proc_work also executes in the conn workq */ + + struct scoutfs_tseq_entry tseq_entry; + + void *info; +}; + +enum conn_flags { + CONN_FL_valid_greeting = (1UL << 0), /* other commands can proceed */ + CONN_FL_established = (1UL << 1), /* added sends queue send work */ + CONN_FL_shutting_down = (1UL << 2), /* shutdown work was queued */ + CONN_FL_saw_greeting = (1UL << 3), /* saw greeting on this sock */ + CONN_FL_saw_farewell = (1UL << 4), /* saw farewell response */ + CONN_FL_reconn_wait = (1UL << 5), /* shutdown, waiting for reconn */ + CONN_FL_reconn_freeing = (1UL << 6), /* waiting done, setter frees */ +}; + +#define SIN_FMT "%pIS:%u" +#define SIN_ARG(sin) sin, be16_to_cpu((sin)->sin_port) + +static inline void scoutfs_addr_to_sin(struct sockaddr_in *sin, + struct scoutfs_inet_addr *addr) +{ + sin->sin_family = AF_INET; + sin->sin_addr.s_addr = cpu_to_be32(le32_to_cpu(addr->addr)); + sin->sin_port = cpu_to_be16(le16_to_cpu(addr->port)); +} + +static inline void scoutfs_addr_from_sin(struct scoutfs_inet_addr *addr, + struct sockaddr_in *sin) +{ + addr->addr = be32_to_le32(sin->sin_addr.s_addr); + addr->port = be16_to_le16(sin->sin_port); + memset(addr->__pad, 0, sizeof(addr->__pad)); +} + +struct scoutfs_net_connection * +scoutfs_net_alloc_conn(struct super_block *sb, + scoutfs_net_notify_t notify_up, + scoutfs_net_notify_t notify_down, size_t info_size, + scoutfs_net_request_t *req_funcs, char *name_suffix); +u64 scoutfs_net_client_rid(struct scoutfs_net_connection *conn); +int scoutfs_net_connect(struct super_block *sb, + struct scoutfs_net_connection *conn, + struct sockaddr_in *sin, unsigned long timeout_ms); +int scoutfs_net_bind(struct super_block *sb, + struct scoutfs_net_connection *conn, + struct sockaddr_in *sin); +void scoutfs_net_listen(struct super_block *sb, + struct scoutfs_net_connection *conn); +int scoutfs_net_submit_request(struct super_block *sb, + struct scoutfs_net_connection *conn, + u8 cmd, void *arg, u16 arg_len, + scoutfs_net_response_t resp_func, + void *resp_data, u64 *id_ret); +int scoutfs_net_submit_request_node(struct super_block *sb, + struct scoutfs_net_connection *conn, + u64 rid, u8 cmd, void *arg, u16 arg_len, + scoutfs_net_response_t resp_func, + void *resp_data, u64 *id_ret); +void scoutfs_net_cancel_request(struct super_block *sb, + struct scoutfs_net_connection *conn, + u8 cmd, u64 id); +int scoutfs_net_sync_request(struct super_block *sb, + struct scoutfs_net_connection *conn, + u8 cmd, void *arg, unsigned arg_len, + void *resp, size_t resp_len); +int scoutfs_net_response(struct super_block *sb, + struct scoutfs_net_connection *conn, + u8 cmd, u64 id, int error, void *resp, u16 resp_len); +int scoutfs_net_response_node(struct super_block *sb, + struct scoutfs_net_connection *conn, + u64 rid, u8 cmd, u64 id, int error, + void *resp, u16 resp_len); +void scoutfs_net_shutdown(struct super_block *sb, + struct scoutfs_net_connection *conn); +void scoutfs_net_free_conn(struct super_block *sb, + struct scoutfs_net_connection *conn); + +void scoutfs_net_client_greeting(struct super_block *sb, + struct scoutfs_net_connection *conn, + bool new_server); +void scoutfs_net_server_greeting(struct super_block *sb, + struct scoutfs_net_connection *conn, + u64 rid, u64 greeting_id, + bool reconnecting, bool first_contact, + bool farewell); +void scoutfs_net_farewell(struct super_block *sb, + struct scoutfs_net_connection *conn); + +int scoutfs_net_setup(struct super_block *sb); +void scoutfs_net_destroy(struct super_block *sb); + +#endif diff --git a/kmod/src/options.c b/kmod/src/options.c new file mode 100644 index 00000000..4d698b36 --- /dev/null +++ b/kmod/src/options.c @@ -0,0 +1,215 @@ +/* + * Copyright (C) 2017 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "msg.h" +#include "options.h" +#include "super.h" + +static const match_table_t tokens = { + {Opt_server_addr, "server_addr=%s"}, + {Opt_metadev_path, "metadev_path=%s"}, + {Opt_err, NULL} +}; + +struct options_sb_info { + struct dentry *debugfs_dir; +}; + +u32 scoutfs_option_u32(struct super_block *sb, int token) +{ + WARN_ON_ONCE(1); + return 0; +} + +/* The caller's string is null terminted and can be clobbered */ +static int parse_ipv4(struct super_block *sb, char *str, + struct sockaddr_in *sin) +{ + unsigned long port = 0; + __be32 addr; + char *c; + int ret; + + /* null term port, if specified */ + c = strchr(str, ':'); + if (c) + *c = '\0'; + + /* parse addr */ + addr = in_aton(str); + if (ipv4_is_multicast(addr) || ipv4_is_lbcast(addr) || + ipv4_is_zeronet(addr) || + ipv4_is_local_multicast(addr)) { + scoutfs_err(sb, "invalid unicast ipv4 address: %s", str); + return -EINVAL; + } + + /* parse port, if specified */ + if (c) { + c++; + ret = kstrtoul(c, 0, &port); + if (ret != 0 || port == 0 || port >= U16_MAX) { + scoutfs_err(sb, "invalid port in ipv4 address: %s", c); + return -EINVAL; + } + } + + sin->sin_family = AF_INET; + sin->sin_addr.s_addr = addr; + sin->sin_port = cpu_to_be16(port); + + return 0; +} + +static int parse_bdev_path(struct super_block *sb, substring_t *substr, + char **bdev_path_ret) +{ + char *bdev_path; + struct inode *bdev_inode; + struct path path; + bool got_path = false; + int ret; + + bdev_path = match_strdup(substr); + if (!bdev_path) { + scoutfs_err(sb, "bdev string dup failed"); + ret = -ENOMEM; + goto out; + } + + ret = kern_path(bdev_path, LOOKUP_FOLLOW, &path); + if (ret) { + scoutfs_err(sb, "path %s not found for bdev: error %d", + bdev_path, ret); + goto out; + } + got_path = true; + + bdev_inode = d_inode(path.dentry); + if (!S_ISBLK(bdev_inode->i_mode)) { + scoutfs_err(sb, "path %s for bdev is not a block device", + bdev_path); + ret = -ENOTBLK; + goto out; + } + +out: + if (got_path) { + path_put(&path); + } + + if (ret < 0) { + kfree(bdev_path); + } else { + *bdev_path_ret = bdev_path; + } + + return ret; +} + +int scoutfs_parse_options(struct super_block *sb, char *options, + struct mount_options *parsed) +{ + char ipstr[INET_ADDRSTRLEN + 1]; + substring_t args[MAX_OPT_ARGS]; + int token; + char *p; + int ret; + + /* Set defaults */ + memset(parsed, 0, sizeof(*parsed)); + + while ((p = strsep(&options, ",")) != NULL) { + if (!*p) + continue; + + token = match_token(p, tokens, args); + switch (token) { + case Opt_server_addr: + + match_strlcpy(ipstr, args, ARRAY_SIZE(ipstr)); + ret = parse_ipv4(sb, ipstr, &parsed->server_addr); + if (ret < 0) + return ret; + break; + case Opt_metadev_path: + + ret = parse_bdev_path(sb, &args[0], + &parsed->metadev_path); + if (ret < 0) + return ret; + break; + default: + scoutfs_err(sb, "Unknown or malformed option, \"%s\"", + p); + break; + } + } + + if (!parsed->metadev_path) { + scoutfs_err(sb, "Required mount option \"metadev_path\" not found"); + return -EINVAL; + } + + return 0; +} + +int scoutfs_options_setup(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct options_sb_info *osi; + int ret; + + osi = kzalloc(sizeof(struct options_sb_info), GFP_KERNEL); + if (!osi) + return -ENOMEM; + + sbi->options = osi; + + osi->debugfs_dir = debugfs_create_dir("options", sbi->debug_root); + if (!osi->debugfs_dir) { + ret = -ENOMEM; + goto out; + } + + ret = 0; +out: + if (ret) + scoutfs_options_destroy(sb); + return ret; +} + +void scoutfs_options_destroy(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct options_sb_info *osi = sbi->options; + + if (osi) { + if (osi->debugfs_dir) + debugfs_remove_recursive(osi->debugfs_dir); + kfree(osi); + sbi->options = NULL; + } +} diff --git a/kmod/src/options.h b/kmod/src/options.h new file mode 100644 index 00000000..b62be4d3 --- /dev/null +++ b/kmod/src/options.h @@ -0,0 +1,27 @@ +#ifndef _SCOUTFS_OPTIONS_H_ +#define _SCOUTFS_OPTIONS_H_ + +#include +#include +#include "format.h" + +enum scoutfs_mount_options { + Opt_server_addr, + Opt_metadev_path, + Opt_err, +}; + +struct mount_options { + struct sockaddr_in server_addr; + char *metadev_path; +}; + +int scoutfs_parse_options(struct super_block *sb, char *options, + struct mount_options *parsed); +int scoutfs_options_setup(struct super_block *sb); +void scoutfs_options_destroy(struct super_block *sb); + +u32 scoutfs_option_u32(struct super_block *sb, int token); +#define scoutfs_option_bool scoutfs_option_u32 + +#endif /* _SCOUTFS_OPTIONS_H_ */ diff --git a/kmod/src/per_task.c b/kmod/src/per_task.c new file mode 100644 index 00000000..c0424e4d --- /dev/null +++ b/kmod/src/per_task.c @@ -0,0 +1,111 @@ +/* + * Copyright (C) 2017 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include + +#include "per_task.h" + +/* + * There are times when we'd like to pass data from a caller to its + * callee but we're bouncing through functions and callbacks that don't + * provide per-task storage. We add a trivial little locked list that + * lets a caller store a pointer for callees. The lists are put in the + * scope of the sharing so the contention is rare and limited to real + * concurrency -- imagine, for example, concurrent file reading on an + * inode. + */ + +/* + * Return the pointer that our caller added for us on the given list. + * The expected promise is that the pointer is valid until we return to + * the caller who will remove it from the list. + */ +void *scoutfs_per_task_get(struct scoutfs_per_task *pt) +{ + const struct task_struct *task = current; + struct scoutfs_per_task_entry *ent; + void *ret = NULL; + + spin_lock(&pt->lock); + + list_for_each_entry(ent, &pt->list, head) { + if (ent->task == task){ + ret = ent->ptr; + break; + } + } + + spin_unlock(&pt->lock); + + return ret; +} + +void scoutfs_per_task_add(struct scoutfs_per_task *pt, + struct scoutfs_per_task_entry *ent, void *ptr) +{ + ent->task = current; + ent->ptr = ptr; + + spin_lock(&pt->lock); + list_add(&ent->head, &pt->list); + spin_unlock(&pt->lock); +} + +/* + * Add the entry to the per-task list if the task didn't already have an + * entry on the list. Returns true if the entry was added, false if it + * wasn't. + */ +bool scoutfs_per_task_add_excl(struct scoutfs_per_task *pt, + struct scoutfs_per_task_entry *ent, void *ptr) +{ + if (!scoutfs_per_task_get(pt)) { + scoutfs_per_task_add(pt, ent, ptr); + return true; + } + + return false; +} + +/* + * Return true if the entry was found on the list and was deleted, + * returns false if the entry wasn't present on a list. + */ +bool scoutfs_per_task_del(struct scoutfs_per_task *pt, + struct scoutfs_per_task_entry *ent) +{ + BUG_ON(!list_empty(&ent->head) && ent->task != current); + + if (!list_empty(&ent->head)) { + spin_lock(&pt->lock); + list_del_init(&ent->head); + spin_unlock(&pt->lock); + return true; + } + + return false; +} + +void scoutfs_per_task_init(struct scoutfs_per_task *pt) +{ + spin_lock_init(&pt->lock); + INIT_LIST_HEAD(&pt->list); +} + +void scoutfs_per_task_init_entry(struct scoutfs_per_task_entry *ent) +{ + INIT_LIST_HEAD(&ent->head); + ent->task = NULL; + ent->ptr = NULL; +} diff --git a/kmod/src/per_task.h b/kmod/src/per_task.h new file mode 100644 index 00000000..38616616 --- /dev/null +++ b/kmod/src/per_task.h @@ -0,0 +1,31 @@ +#ifndef _SCOUTFS_PER_TASK_H_ +#define _SCOUTFS_PER_TASK_H_ + +struct scoutfs_per_task { + spinlock_t lock; + struct list_head list; +}; + +struct scoutfs_per_task_entry { + struct list_head head; + struct task_struct *task; + void *ptr; +}; + +#define SCOUTFS_DECLARE_PER_TASK_ENTRY(name) \ + struct scoutfs_per_task_entry name = { \ + .head = LIST_HEAD_INIT((name).head), \ + } + + +void *scoutfs_per_task_get(struct scoutfs_per_task *pt); +void scoutfs_per_task_add(struct scoutfs_per_task *pt, + struct scoutfs_per_task_entry *ent, void *ptr); +bool scoutfs_per_task_add_excl(struct scoutfs_per_task *pt, + struct scoutfs_per_task_entry *ent, void *ptr); +bool scoutfs_per_task_del(struct scoutfs_per_task *pt, + struct scoutfs_per_task_entry *ent); +void scoutfs_per_task_init(struct scoutfs_per_task *pt); +void scoutfs_per_task_init_entry(struct scoutfs_per_task_entry *ent); + +#endif diff --git a/kmod/src/quorum.c b/kmod/src/quorum.c new file mode 100644 index 00000000..43c398d9 --- /dev/null +++ b/kmod/src/quorum.c @@ -0,0 +1,779 @@ +/* + * Copyright (C) 2019 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "format.h" +#include "msg.h" +#include "counters.h" +#include "quorum.h" +#include "server.h" +#include "net.h" +#include "sysfs.h" +#include "scoutfs_trace.h" + +/* + * scoutfs mounts communicate through a region of preallocated blocks to + * elect a leader who starts the server. Mounts which have been + * configured with a server address and which can't connect to a server + * attempt to form a quorum to elect a new leader who starts a new + * server. + * + * The mounts participating in the election use a variant of the raft + * election protocol to establish quorum and elect a leader. We use + * block reads and writes instead of network messages. Mounts read all + * the blocks looking for messages to receive. Mounts write their vote + * to a random block in the region to send a message to all other + * mounts. Unlikely collisions are analogous to lossy networks losing + * messages and are handled by the protocol. + * + * We allow a "majority" of 1 voter when there are less than three + * possible voters. This lets a simple network establish quorum. If + * the raft quorum timeouts align to leaders could both elect themselves + * and race to fence each other. In the worst case they could continue + * to do this indefinitely but it's unlikely as it would require a + * sequence of identical random raft timeouts. + * + * One of the reasons we use block reads and writes as the quorum + * communication medium is that it lets us leave behind a shared + * persistent log of previous election results. This then lets a newly + * elected leader fence all previously elected leaders that haven't + * shutdown so that they can safely assume exclusive access to the + * shared device. Every written block includes a log of election + * results. Every voter merges the log from every block it reads the + * block it writes. A leader doesn't attempt to fence until it's spent + * a few cycles writing blocks with itself as the log entry. This gives + * other voters time to migrate the log entry through the blocks. + * + * Once a leader is elected it fences any previously elected leaders + * still present in the log it merged while reading all the voting + * blocks. Once they've fenced they update the super block record of + * the latest term that has been fenced. This trims the log over time + * and keeps from attempting to fence the same mounts multiple times. + * As the server later shuts down it writes its term into the super to + * stop it from being fenced. + * + * The final complication comes during unmount. Clients exit after the + * server responds to their farewell request. But a majority of clients + * need to be present to elect a server to process farewell requests. + * The server knows which clients will attempt to vote for quorum and + * only responds to their farewell requests once they're no longer + * needed to elect a server -- either there's still quorum remaining of + * other mounts or the only mounts remaining are all quorum voters that + * have sent farewell requests. Before sending these final responses + * the server updates an unmount_barrier field in the super. If clients + * that are waiting for a farewell response see the unmount barrier + * increment they know that their farewell has been processed and they + * can assume a successful farewell response and exit cleanly. + * + * XXX: - actually fence + */ + +struct quorum_info { + struct scoutfs_sysfs_attrs ssa; + + bool is_leader; +}; + +#define DECLARE_QUORUM_INFO(sb, name) \ + struct quorum_info *name = SCOUTFS_SB(sb)->quorum_info +#define DECLARE_QUORUM_INFO_KOBJ(kobj, name) \ + DECLARE_QUORUM_INFO(SCOUTFS_SYSFS_ATTRS_SB(kobj), name) + +/* + * Return an absolute ktime timeout expires value in the future after a + * random duration between hi and lo where both limits are possible. + */ +static ktime_t random_to(u32 lo, u32 hi) +{ + return ktime_add_ms(ktime_get(), lo + prandom_u32_max((hi + 1) - lo)); +} + +/* + * The caller is about to read all the quorum blocks. We invalidate any + * cached blocks and issue one large contiguous read to repopulate the + * cache. The caller then uses normal __bread to read each block. I'm + * not a huge fan of the plug but I couldn't get the individual + * readahead requests merged without it. + */ +static void readahead_quorum_blocks(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct buffer_head *bh; + struct blk_plug plug; + int i; + + blk_start_plug(&plug); + + for (i = 0; i < SCOUTFS_QUORUM_BLOCKS; i++) { + bh = __getblk(sbi->meta_bdev, SCOUTFS_QUORUM_BLKNO + i, + SCOUTFS_BLOCK_SM_SIZE); + if (!bh) + continue; + + lock_buffer(bh); + clear_buffer_uptodate(bh); + unlock_buffer(bh); + + ll_rw_block(READA | REQ_META | REQ_PRIO, 1, &bh); + brelse(bh); + } + + blk_finish_plug(&plug); +} + +struct quorum_block_head { + struct list_head head; + union { + struct scoutfs_quorum_block blk; + u8 bytes[SCOUTFS_BLOCK_SM_SIZE]; + }; +}; + +static void free_quorum_blocks(struct list_head *blocks) +{ + struct quorum_block_head *qbh; + struct quorum_block_head *tmp; + + list_for_each_entry_safe(qbh, tmp, blocks, head) { + list_del_init(&qbh->head); + kfree(qbh); + } +} + +/* + * Callers don't mind us clobbering the crc temporarily. + */ +static __le32 quorum_block_crc(struct scoutfs_quorum_block *blk) +{ + __le32 calc_crc; + __le32 blk_crc; + + blk_crc = blk->crc; + blk->crc = 0; + calc_crc = cpu_to_le32(crc32c(~0, blk, sizeof(*blk))); + blk->crc = blk_crc; + + return calc_crc; +} + +static size_t quorum_block_bytes(struct scoutfs_quorum_block *blk) +{ + return offsetof(struct scoutfs_quorum_block, + log[blk->log_nr]); +} + +static bool invalid_quorum_block(struct buffer_head *bh, + struct scoutfs_quorum_block *blk) +{ + return bh->b_size != SCOUTFS_BLOCK_SM_SIZE || + sizeof(struct scoutfs_quorum_block) > SCOUTFS_BLOCK_SM_SIZE || + quorum_block_crc(blk) != blk->crc || + le64_to_cpu(blk->blkno) != bh->b_blocknr || + blk->term == 0 || + blk->log_nr > SCOUTFS_QUORUM_LOG_MAX || + quorum_block_bytes(blk) > SCOUTFS_BLOCK_SM_SIZE; +} + +/* true if a is stale and should be ignored */ +static bool stale_quorum_block(struct scoutfs_quorum_block *a, + struct scoutfs_quorum_block *b) +{ + if (le64_to_cpu(a->term) < le64_to_cpu(b->term)) + return true; + + if (le64_to_cpu(a->voter_rid) == le64_to_cpu(b->voter_rid) && + le64_to_cpu(a->write_nr) <= le64_to_cpu(b->write_nr)) + return true; + + return false; +} + +/* + * Get the most recent blocks from all the voters for the most recent term. + * We ignore any corrupt blocks, blocks not for our fsid, previous terms, + * and previous writes from a rid in the current term. + */ +static int read_quorum_blocks(struct super_block *sb, struct list_head *blocks) +{ + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_quorum_block *blk; + struct quorum_block_head *qbh; + struct quorum_block_head *tmp; + struct buffer_head *bh = NULL; + LIST_HEAD(stale); + int ret; + int i; + + readahead_quorum_blocks(sb); + + for (i = 0; i < SCOUTFS_QUORUM_BLOCKS; i++) { + brelse(bh); + bh = __bread(sbi->meta_bdev, SCOUTFS_QUORUM_BLKNO + i, + SCOUTFS_BLOCK_SM_SIZE); + if (!bh) { + scoutfs_inc_counter(sb, quorum_read_block_error); + ret = -EIO; + goto out; + } + blk = (void *)(bh->b_data); + + /* ignore unwritten blocks or blocks for other filesystems */ + if (blk->voter_rid == 0 || blk->fsid != super->hdr.fsid) + continue; + + if (invalid_quorum_block(bh, blk)) { + scoutfs_inc_counter(sb, quorum_read_invalid_block); + continue; + } + + list_for_each_entry_safe(qbh, tmp, blocks, head) { + if (stale_quorum_block(blk, &qbh->blk)) { + blk = NULL; + break; + } + + if (stale_quorum_block(&qbh->blk, blk)) + list_move(&qbh->head, &stale); + } + free_quorum_blocks(&stale); + + if (!blk) + continue; + + qbh = kmalloc(sizeof(struct quorum_block_head), + GFP_NOFS); + if (!qbh) { + ret = -ENOMEM; + goto out; + } + + memcpy(&qbh->blk, blk, quorum_block_bytes(blk)); + list_add_tail(&qbh->head, blocks); + } + + list_for_each_entry(qbh, blocks, head) { + trace_scoutfs_quorum_read_block(sb, &qbh->blk); + scoutfs_inc_counter(sb, quorum_read_block); + } + + ret = 0; +out: + brelse(bh); + if (ret < 0) + free_quorum_blocks(blocks); + return ret; +} + +/* + * Synchronously write a single quorum block. The caller has provided + * the meaningful fields for the write. We fill in the fsid, blkno, and + * crc for every write and zero the rest of the block. + */ +static int write_quorum_block(struct super_block *sb, + struct scoutfs_quorum_block *our_blk) +{ + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_quorum_block *blk; + struct buffer_head *bh = NULL; + size_t size; + int ret; + + BUILD_BUG_ON(sizeof(struct scoutfs_quorum_block) > + SCOUTFS_BLOCK_SM_SIZE); + + bh = __getblk(sbi->meta_bdev, SCOUTFS_QUORUM_BLKNO + + prandom_u32_max(SCOUTFS_QUORUM_BLOCKS), + SCOUTFS_BLOCK_SM_SIZE); + if (bh == NULL) { + ret = -EIO; + goto out; + } + + size = quorum_block_bytes(our_blk); + if (WARN_ON_ONCE(size > SCOUTFS_BLOCK_SM_SIZE || size > bh->b_size)) { + ret = -EIO; + goto out; + } + + blk = (void *)bh->b_data; + memset(blk, 0, bh->b_size); + memcpy(blk, our_blk, size); + + blk->fsid = super->hdr.fsid; + blk->blkno = cpu_to_le64(bh->b_blocknr); + blk->crc = quorum_block_crc(blk); + + lock_buffer(bh); + set_buffer_mapped(bh); + bh->b_end_io = end_buffer_write_sync; + get_bh(bh); + submit_bh(WRITE_SYNC | REQ_META | REQ_PRIO, bh); + + wait_on_buffer(bh); + if (!buffer_uptodate(bh)) + ret = -EIO; + else + ret = 0; + + if (ret == 0) { + trace_scoutfs_quorum_write_block(sb, blk); + scoutfs_inc_counter(sb, quorum_write_block); + } +out: + if (ret) + scoutfs_inc_counter(sb, quorum_write_block_error); + brelse(bh); + return ret; +} + +/* + * Returns true if there's an entry for the given election. + */ +static bool log_contains(struct scoutfs_quorum_block *blk, u64 term, u64 rid) +{ + int i; + + for (i = 0; i < blk->log_nr; i++) { + if (le64_to_cpu(blk->log[i].term) == term && + le64_to_cpu(blk->log[i].rid) == rid) + return true; + } + + return false; +} + +/* add an entry to the log, returning error if it's full */ +static int log_add(struct scoutfs_quorum_block *blk, u64 term, u64 rid, + struct scoutfs_inet_addr *addr) +{ + int i; + + if (log_contains(blk, term, rid)) + return 0; + + if (blk->log_nr == SCOUTFS_QUORUM_LOG_MAX) + return -ENOSPC; + + i = blk->log_nr++; + blk->log[i].term = cpu_to_le64(term); + blk->log[i].rid = cpu_to_le64(rid); + blk->log[i].addr = *addr; + + return 0; +} + +/* migrate live log entries between blocks, returning err if full */ +static int log_merge(struct scoutfs_quorum_block *our_blk, + struct scoutfs_quorum_block *blk, + u64 fenced_term) +{ + int ret; + int i; + + for (i = 0; i < blk->log_nr; i++) { + if (le64_to_cpu(blk->log[i].term) > fenced_term) { + ret = log_add(our_blk, le64_to_cpu(blk->log[i].term), + le64_to_cpu(blk->log[i].rid), + &blk->log[i].addr); + if (ret < 0) + return ret; + } + } + + return 0; +} + +/* Remove old log entries for a voter before a given term. */ +static void log_purge(struct scoutfs_quorum_block *blk, u64 term, u64 rid) +{ + int i; + + for (i = 0; i < blk->log_nr; i++) { + if (le64_to_cpu(blk->log[i].term) < term && + le64_to_cpu(blk->log[i].rid) == rid) { + if (i != blk->log_nr - 1) + swap(blk->log[i], blk->log[blk->log_nr - 1]); + blk->log_nr--; + i--; /* continue from swapped in entry */ + } + } +} + + +/* + * The caller received a majority of votes and has been elected. Before + * assuming exclusive write access to the device we fence the winners of + * any previous elections still present in the log. Once they're fenced + * we re-read the super and update the fenced_term to indicate that + * those previous elections can be ignored and purged from the log. + * + * We can be attempting this concurrently with both previous and future + * elected leaders. The leader with the greatest elected term will win + * and fence all previous elected leaders. + * + * We clobber the caller's block as we go to not fence rids multiple times. + */ +static int fence_previous(struct super_block *sb, + struct scoutfs_quorum_block *blk, + u64 our_rid, u64 fenced_term, u64 term) +{ + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + struct sockaddr_in their_sin; + int ret; + int i; + + for (i = 0; i < blk->log_nr; i++) { + if (le64_to_cpu(blk->log[i].rid) != our_rid && + le64_to_cpu(blk->log[i].term) > fenced_term && + le64_to_cpu(blk->log[i].term) < term) { + + scoutfs_inc_counter(sb, quorum_fenced); + scoutfs_addr_to_sin(&their_sin, &blk->log[i].addr); + scoutfs_err(sb, "fencing "SCSBF" at "SIN_FMT, + SCSB_LEFR_ARGS(super->hdr.fsid, + blk->log[i].rid), + SIN_ARG(&their_sin)); + + log_purge(blk, term, le64_to_cpu(blk->log[i].rid)); + i = -1; /* start over */ + } + } + + /* update fenced term now that we have exclusive access */ + ret = 0; + super = kmalloc(sizeof(struct scoutfs_super_block), GFP_NOFS); + if (super) { + ret = scoutfs_read_super(sb, super); + if (ret == 0) { + super->quorum_fenced_term = cpu_to_le64(term - 1); + ret = scoutfs_write_super(sb, super); + + } + kfree(super); + } else { + ret = -ENOMEM; + } + + if (ret != 0) { + scoutfs_err(sb, "failed to update fenced_term in super, this mount will probably be fenced"); + } + + return ret; +} + + + +/* + * The calling voting mount couldn't connect to a server. Participate + * in a raft election to chose a mount to start a new server. If a + * majority of other mounts join us then one of us will be elected and + * our caller will start the server. + * + * Voting members read the blocks at regular intervals. If they see a + * new election they vote for that candidate for the remainder of the + * election. If the election timeout expires they will start a new + * election and vote for themselves. Eventually a sufficient majority + * sees a new election and all vote in the majority for that candidate. + * + * The calling client may have just failed to connect to an elected + * address in the super block. We assume that server is dead and ignore + * it when trying to elect a new leader. But we eventually return with + * a timeout because the server could actually be fine and the client + * could have had communication to the server restored. + * + * We return success if we see a new server elected. If we are elected + * we set the caller's elected_term so they know to start the server. + */ +int scoutfs_quorum_election(struct super_block *sb, ktime_t timeout_abs, + u64 prev_term, u64 *elected_term) +{ + DECLARE_QUORUM_INFO(sb, qinf); + struct mount_options *opts = &SCOUTFS_SB(sb)->opts; + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = NULL; + struct scoutfs_quorum_block *our_blk = NULL; + struct scoutfs_quorum_block *blk; + struct quorum_block_head *qbh; + struct scoutfs_inet_addr addr; + enum { VOTER, CANDIDATE }; + ktime_t cycle_to; + ktime_t term_to; + LIST_HEAD(blocks); + u64 vote_for_write_nr; + u64 vote_for_rid; + u64 write_nr; + u64 term; + int log_cycles = 0; + int votes; + int role; + int ret; + + *elected_term = 0; + + trace_scoutfs_quorum_election(sb, prev_term); + + super = kmalloc(sizeof(struct scoutfs_super_block), GFP_NOFS); + our_blk = kmalloc(SCOUTFS_BLOCK_SM_SIZE, GFP_NOFS); + if (!super || !our_blk) { + ret = -ENOMEM; + goto out; + } + + /* start out as a passive voter */ + role = VOTER; + term = 0; + write_nr = 0; + vote_for_rid = 0; + vote_for_write_nr = 0; + + /* we'll become a candidate if we don't see another candidate */ + term_to = random_to(SCOUTFS_QUORUM_TERM_LO_MS, + SCOUTFS_QUORUM_TERM_HI_MS); + + for (;;) { + memset(our_blk, 0, SCOUTFS_BLOCK_SM_SIZE); + + scoutfs_inc_counter(sb, quorum_cycle); + + ret = scoutfs_read_super(sb, super); + if (ret) + goto out; + + /* done if we see evidence of a new server */ + if (le64_to_cpu(super->quorum_server_term) > prev_term) { + scoutfs_inc_counter(sb, quorum_saw_super_leader); + ret = 0; + goto out; + } + + /* done if we couldn't elect anyone */ + if (ktime_after(ktime_get(), timeout_abs)) { + scoutfs_inc_counter(sb, quorum_timedout); + ret = -ETIMEDOUT; + goto out; + } + + /* become a candidate if the election times out */ + if (ktime_after(ktime_get(), term_to)) { + scoutfs_inc_counter(sb, quorum_election_timeout); + term_to = random_to(SCOUTFS_QUORUM_TERM_LO_MS, + SCOUTFS_QUORUM_TERM_HI_MS); + role = CANDIDATE; + term++; + vote_for_rid = sbi->rid; + log_cycles = 0; + } + + free_quorum_blocks(&blocks); + ret = read_quorum_blocks(sb, &blocks); + if (ret < 0) + goto out; + + votes = 0; + + list_for_each_entry(qbh, &blocks, head) { + blk = &qbh->blk; + + /* + * Become a voter for a candidate the first time + * we see a new term. + * + * And also if we're a candidate and see a + * higher rid candidate in our term. This + * minimizes instability when two quorums are + * possible and race to elect two leaders. This + * is only barely reasonable when accepting the + * risk of instability in two mount + * configurations. + */ + if ((le64_to_cpu(blk->term) > term) || + (role == CANDIDATE && + le64_to_cpu(blk->term) == term && + blk->voter_rid == blk->vote_for_rid && + le64_to_cpu(blk->voter_rid) > sbi->rid)) { + role = VOTER; + term = le64_to_cpu(blk->term); + vote_for_rid = le64_to_cpu(blk->vote_for_rid); + vote_for_write_nr = 0; + votes = 0; + log_cycles = 0; + } + + /* candidate writes suppress voter election timers */ + if (role == VOTER && + blk->voter_rid == blk->vote_for_rid && + le64_to_cpu(blk->write_nr) > vote_for_write_nr) { + term_to = random_to(SCOUTFS_QUORUM_TERM_LO_MS, + SCOUTFS_QUORUM_TERM_HI_MS); + vote_for_write_nr = le64_to_cpu(blk->write_nr); + } + + /* count our votes */ + if (role == CANDIDATE && + le64_to_cpu(blk->vote_for_rid) == sbi->rid) { + votes++; + } + + /* try to write greater write_nr */ + write_nr = max(write_nr, le64_to_cpu(blk->write_nr)); + } + + trace_scoutfs_quorum_election_vote(sb, role, term, + vote_for_rid, votes, + log_cycles, + super->quorum_count); + + /* first merge logs from all votes this term */ + list_for_each_entry(qbh, &blocks, head) { + blk = &qbh->blk; + + ret = log_merge(our_blk, blk, + le64_to_cpu(super->quorum_fenced_term)); + if (ret < 0) + goto out; + } + + /* remove logs for voters that can't be servers */ + list_for_each_entry(qbh, &blocks, head) { + blk = &qbh->blk; + + if (blk->voter_rid != blk->vote_for_rid) + log_purge(our_blk, le64_to_cpu(blk->term), + le64_to_cpu(blk->voter_rid)); + } + + /* add ourselves to the log when we see vote quorum */ + if (role == CANDIDATE && votes >= super->quorum_count) { + scoutfs_addr_from_sin(&addr, &opts->server_addr); + ret = log_add(our_blk, term, vote_for_rid, &addr); + if (ret < 0) + goto out; + log_cycles++; /* will be written *this* cycle */ + } + + /* elected candidates can proceed after their log cycles */ + if (role == CANDIDATE && + log_cycles > SCOUTFS_QUORUM_ELECTED_LOG_CYCLES) { + /* our_blk is clobbered */ + ret = fence_previous(sb, our_blk, sbi->rid, + le64_to_cpu(super->quorum_fenced_term), + term); + if (ret < 0) + goto out; + scoutfs_inc_counter(sb, quorum_elected_leader); + qinf->is_leader = true; + *elected_term = term; + goto out; + } + + /* write our block every cycle */ + if (term > 0) { + our_blk->term = cpu_to_le64(term); + write_nr++; + our_blk->write_nr = cpu_to_le64(write_nr); + our_blk->voter_rid = cpu_to_le64(sbi->rid); + our_blk->vote_for_rid = cpu_to_le64(vote_for_rid); + + ret = write_quorum_block(sb, our_blk); + if (ret < 0) + goto out; + } + + /* add a small random delay to each cycle */ + cycle_to = random_to(SCOUTFS_QUORUM_CYCLE_LO_MS, + SCOUTFS_QUORUM_CYCLE_HI_MS); + set_current_state(TASK_UNINTERRUPTIBLE); + schedule_hrtimeout(&cycle_to, HRTIMER_MODE_ABS); + } + +out: + free_quorum_blocks(&blocks); + kfree(super); + kfree(our_blk); + + trace_scoutfs_quorum_election_ret(sb, ret, *elected_term); + if (ret) + scoutfs_inc_counter(sb, quorum_failure); + + return ret; +} + +void scoutfs_quorum_clear_leader(struct super_block *sb) +{ + DECLARE_QUORUM_INFO(sb, qinf); + + qinf->is_leader = false; +} + +static ssize_t is_leader_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + DECLARE_QUORUM_INFO_KOBJ(kobj, qinf); + + return snprintf(buf, PAGE_SIZE, "%u", !!qinf->is_leader); +} +SCOUTFS_ATTR_RO(is_leader); + +static struct attribute *quorum_attrs[] = { + SCOUTFS_ATTR_PTR(is_leader), + NULL, +}; + +int scoutfs_quorum_setup(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct quorum_info *qinf; + int ret; + + qinf = kzalloc(sizeof(struct quorum_info), GFP_KERNEL); + if (!qinf) { + ret = -ENOMEM; + goto out; + } + scoutfs_sysfs_init_attrs(sb, &qinf->ssa); + + sbi->quorum_info = qinf; + + ret = scoutfs_sysfs_create_attrs(sb, &qinf->ssa, quorum_attrs, + "quorum"); +out: + if (ret) + scoutfs_quorum_destroy(sb); + + return 0; +} + +void scoutfs_quorum_destroy(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct quorum_info *qinf = SCOUTFS_SB(sb)->quorum_info; + + if (qinf) { + scoutfs_sysfs_destroy_attrs(sb, &qinf->ssa); + kfree(qinf); + sbi->quorum_info = NULL; + } +} diff --git a/kmod/src/quorum.h b/kmod/src/quorum.h new file mode 100644 index 00000000..96eac0e4 --- /dev/null +++ b/kmod/src/quorum.h @@ -0,0 +1,10 @@ +#ifndef _SCOUTFS_QUORUM_H_ +#define _SCOUTFS_QUORUM_H_ + +int scoutfs_quorum_election(struct super_block *sb, ktime_t timeout_abs, + u64 prev_term, u64 *elected_term); +void scoutfs_quorum_clear_leader(struct super_block *sb); + +int scoutfs_quorum_setup(struct super_block *sb); +void scoutfs_quorum_destroy(struct super_block *sb); +#endif diff --git a/kmod/src/scoutfs_trace.c b/kmod/src/scoutfs_trace.c new file mode 100644 index 00000000..6c775b9f --- /dev/null +++ b/kmod/src/scoutfs_trace.c @@ -0,0 +1,28 @@ +/* + * Copyright (C) 2016 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include +#include +#include +#include +#include + +#include "super.h" +#include "format.h" +#include "inode.h" +#include "dir.h" +#include "msg.h" + +#define CREATE_TRACE_POINTS +#include "scoutfs_trace.h" diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h new file mode 100644 index 00000000..aa376dab --- /dev/null +++ b/kmod/src/scoutfs_trace.h @@ -0,0 +1,2471 @@ +/* + * Copyright (C) 2016 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +/* + * This has a crazy name because it's in an external module build at + * the moment. When it's merged upstream it'll move to + * include/trace/events/scoutfs.h + */ + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM scoutfs + +#if !defined(_TRACE_SCOUTFS_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_SCOUTFS_H + +#include +#include +#include + +#include "key.h" +#include "format.h" +#include "lock.h" +#include "super.h" +#include "ioctl.h" +#include "count.h" +#include "export.h" +#include "dir.h" +#include "server.h" +#include "net.h" +#include "data.h" +#include "ext.h" + +struct lock_info; + +#define STE_FMT "[%llu %llu %llu 0x%x]" +#define STE_ARGS(te) (te)->start, (te)->len, (te)->map, (te)->flags +#define STE_FIELDS(pref) \ + __field(__u64, pref##_start) \ + __field(__u64, pref##_len) \ + __field(__u64, pref##_map) \ + __field(__u8, pref##_flags) +#define STE_ASSIGN(pref, te) \ + __entry->pref##_start = (te)->start; \ + __entry->pref##_len = (te)->len; \ + __entry->pref##_map = (te)->map; \ + __entry->pref##_flags = (te)->flags; +#define STE_ENTRY_ARGS(pref) \ + __entry->pref##_start, \ + __entry->pref##_len, \ + __entry->pref##_map, \ + __entry->pref##_flags + +#define DECLARE_TRACED_EXTENT(name) \ + struct scoutfs_traced_extent name = {0} + +DECLARE_EVENT_CLASS(scoutfs_ino_ret_class, + TP_PROTO(struct super_block *sb, u64 ino, int ret), + + TP_ARGS(sb, ino, ret), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(__u64, ino) + __field(int, ret) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->ino = ino; + __entry->ret = ret; + ), + + TP_printk(SCSBF" ino %llu ret %d", + SCSB_TRACE_ARGS, __entry->ino, __entry->ret) +); + +TRACE_EVENT(scoutfs_setattr, + TP_PROTO(struct dentry *dentry, struct iattr *attr), + + TP_ARGS(dentry, attr), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(__u64, ino) + __field(unsigned int, d_len) + __string(d_name, dentry->d_name.name) + __field(__u64, i_size) + __field(__u64, ia_size) + __field(unsigned int, ia_valid) + __field(int, size_change) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(dentry->d_inode->i_sb); + __entry->ino = scoutfs_ino(dentry->d_inode); + __entry->d_len = dentry->d_name.len; + __assign_str(d_name, dentry->d_name.name); + __entry->ia_valid = attr->ia_valid; + __entry->size_change = !!(attr->ia_valid & ATTR_SIZE); + __entry->ia_size = attr->ia_size; + __entry->i_size = i_size_read(dentry->d_inode); + ), + + TP_printk(SCSBF" %s ino %llu ia_valid 0x%x size change %d ia_size " + "%llu i_size %llu", SCSB_TRACE_ARGS, __get_str(d_name), + __entry->ino, __entry->ia_valid, __entry->size_change, + __entry->ia_size, __entry->i_size) +); + +TRACE_EVENT(scoutfs_complete_truncate, + TP_PROTO(struct inode *inode, __u32 flags), + + TP_ARGS(inode, flags), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(__u64, ino) + __field(__u64, i_size) + __field(__u32, flags) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(inode->i_sb); + __entry->ino = scoutfs_ino(inode); + __entry->i_size = i_size_read(inode); + __entry->flags = flags; + ), + + TP_printk(SCSBF" ino %llu i_size %llu flags 0x%x", + SCSB_TRACE_ARGS, __entry->ino, __entry->i_size, + __entry->flags) +); + +TRACE_EVENT(scoutfs_data_fallocate, + TP_PROTO(struct super_block *sb, u64 ino, int mode, loff_t offset, + loff_t len, int ret), + + TP_ARGS(sb, ino, mode, offset, len, ret), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(__u64, ino) + __field(int, mode) + __field(__u64, offset) + __field(__u64, len) + __field(int, ret) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->ino = ino; + __entry->mode = mode; + __entry->offset = offset; + __entry->len = len; + __entry->ret = ret; + ), + + TP_printk(SCSBF" ino %llu mode 0x%x offset %llu len %llu ret %d", + SCSB_TRACE_ARGS, __entry->ino, __entry->mode, __entry->offset, + __entry->len, __entry->ret) +); + +TRACE_EVENT(scoutfs_data_fiemap, + TP_PROTO(struct super_block *sb, __u64 start, __u64 len, int ret), + + + TP_ARGS(sb, start, len, ret), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(__u64, start) + __field(__u64, len) + __field(int, ret) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->start = start; + __entry->len = len; + __entry->ret = ret; + ), + + TP_printk(SCSBF" start %llu len %llu ret %d", SCSB_TRACE_ARGS, + __entry->start, __entry->len, __entry->ret) +); + +TRACE_EVENT(scoutfs_get_block, + TP_PROTO(struct super_block *sb, __u64 ino, __u64 iblock, + int create, struct scoutfs_extent *ext, + int ret, __u64 blkno, size_t size), + + TP_ARGS(sb, ino, iblock, create, ext, ret, blkno, size), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(__u64, ino) + __field(__u64, iblock) + __field(int, create) + STE_FIELDS(ext) + __field(int, ret) + __field(__u64, blkno) + __field(size_t, size) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->ino = ino; + __entry->iblock = iblock; + __entry->create = create; + STE_ASSIGN(ext, ext) + __entry->ret = ret; + __entry->blkno = blkno; + __entry->size = size; + ), + + TP_printk(SCSBF" ino %llu iblock %llu create %d ext "STE_FMT" ret %d bnr %llu size %zu", + SCSB_TRACE_ARGS, __entry->ino, __entry->iblock, + __entry->create, STE_ENTRY_ARGS(ext), __entry->ret, + __entry->blkno, __entry->size) +); + +TRACE_EVENT(scoutfs_data_alloc_block_enter, + TP_PROTO(struct super_block *sb, __u64 ino, __u64 iblock, + struct scoutfs_extent *ext), + + TP_ARGS(sb, ino, iblock, ext), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(__u64, ino) + __field(__u64, iblock) + STE_FIELDS(ext) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->ino = ino; + __entry->iblock = iblock; + STE_ASSIGN(ext, ext) + ), + + TP_printk(SCSBF" ino %llu iblock %llu ext "STE_FMT, + SCSB_TRACE_ARGS, __entry->ino, __entry->iblock, + STE_ENTRY_ARGS(ext)) +); + +DECLARE_EVENT_CLASS(scoutfs_data_file_extent_class, + TP_PROTO(struct super_block *sb, __u64 ino, struct scoutfs_extent *ext), + + TP_ARGS(sb, ino, ext), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(__u64, ino) + STE_FIELDS(ext) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->ino = ino; + STE_ASSIGN(ext, ext) + ), + + TP_printk(SCSBF" ino %llu ext "STE_FMT, + SCSB_TRACE_ARGS, __entry->ino, STE_ENTRY_ARGS(ext)) +); +DEFINE_EVENT(scoutfs_data_file_extent_class, scoutfs_data_alloc, + TP_PROTO(struct super_block *sb, __u64 ino, struct scoutfs_extent *ext), + TP_ARGS(sb, ino, ext) +); +DEFINE_EVENT(scoutfs_data_file_extent_class, scoutfs_data_prealloc, + TP_PROTO(struct super_block *sb, __u64 ino, struct scoutfs_extent *ext), + TP_ARGS(sb, ino, ext) +); +DEFINE_EVENT(scoutfs_data_file_extent_class, scoutfs_data_get_block_found, + TP_PROTO(struct super_block *sb, __u64 ino, struct scoutfs_extent *ext), + TP_ARGS(sb, ino, ext) +); +DEFINE_EVENT(scoutfs_data_file_extent_class, scoutfs_data_get_block_mapped, + TP_PROTO(struct super_block *sb, __u64 ino, struct scoutfs_extent *ext), + TP_ARGS(sb, ino, ext) +); +DEFINE_EVENT(scoutfs_data_file_extent_class, scoutfs_data_extent_truncated, + TP_PROTO(struct super_block *sb, __u64 ino, struct scoutfs_extent *ext), + TP_ARGS(sb, ino, ext) +); +DEFINE_EVENT(scoutfs_data_file_extent_class, scoutfs_data_fiemap_extent, + TP_PROTO(struct super_block *sb, __u64 ino, struct scoutfs_extent *ext), + TP_ARGS(sb, ino, ext) +); + +TRACE_EVENT(scoutfs_data_truncate_items, + TP_PROTO(struct super_block *sb, __u64 iblock, __u64 last, int offline), + + TP_ARGS(sb, iblock, last, offline), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(__u64, iblock) + __field(__u64, last) + __field(int, offline) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->iblock = iblock; + __entry->last = last; + __entry->offline = offline; + ), + + TP_printk(SCSBF" iblock %llu last %llu offline %u", SCSB_TRACE_ARGS, + __entry->iblock, __entry->last, __entry->offline) +); + +TRACE_EVENT(scoutfs_data_wait_check, + TP_PROTO(struct super_block *sb, __u64 ino, __u64 pos, __u64 len, + __u8 sef, __u8 op, struct scoutfs_extent *ext, int ret), + + TP_ARGS(sb, ino, pos, len, sef, op, ext, ret), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(__u64, ino) + __field(__u64, pos) + __field(__u64, len) + __field(__u8, sef) + __field(__u8, op) + STE_FIELDS(ext) + __field(int, ret) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->ino = ino; + __entry->pos = pos; + __entry->len = len; + __entry->sef = sef; + __entry->op = op; + STE_ASSIGN(ext, ext) + __entry->ret = ret; + ), + + TP_printk(SCSBF" ino %llu pos %llu len %llu sef 0x%x op 0x%x ext "STE_FMT" ret %d", + SCSB_TRACE_ARGS, __entry->ino, __entry->pos, __entry->len, + __entry->sef, __entry->op, STE_ENTRY_ARGS(ext), __entry->ret) +); + +TRACE_EVENT(scoutfs_sync_fs, + TP_PROTO(struct super_block *sb, int wait), + + TP_ARGS(sb, wait), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(int, wait) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->wait = wait; + ), + + TP_printk(SCSBF" wait %d", SCSB_TRACE_ARGS, __entry->wait) +); + +TRACE_EVENT(scoutfs_trans_write_func, + TP_PROTO(struct super_block *sb, unsigned long dirty), + + TP_ARGS(sb, dirty), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(unsigned long, dirty) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->dirty = dirty; + ), + + TP_printk(SCSBF" dirty %lu", SCSB_TRACE_ARGS, __entry->dirty) +); + +TRACE_EVENT(scoutfs_release_trans, + TP_PROTO(struct super_block *sb, void *rsv, unsigned int rsv_holders, + struct scoutfs_item_count *res, + struct scoutfs_item_count *act, unsigned int tri_holders, + unsigned int tri_writing, unsigned int tri_items, + unsigned int tri_vals), + + TP_ARGS(sb, rsv, rsv_holders, res, act, tri_holders, tri_writing, + tri_items, tri_vals), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(void *, rsv) + __field(unsigned int, rsv_holders) + __field(int, res_items) + __field(int, res_vals) + __field(int, act_items) + __field(int, act_vals) + __field(unsigned int, tri_holders) + __field(unsigned int, tri_writing) + __field(unsigned int, tri_items) + __field(unsigned int, tri_vals) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->rsv = rsv; + __entry->rsv_holders = rsv_holders; + __entry->res_items = res->items; + __entry->res_vals = res->vals; + __entry->act_items = act->items; + __entry->act_vals = act->vals; + __entry->tri_holders = tri_holders; + __entry->tri_writing = tri_writing; + __entry->tri_items = tri_items; + __entry->tri_vals = tri_vals; + ), + + TP_printk(SCSBF" rsv %p holders %u reserved %u.%u actual " + "%d.%d, trans holders %u writing %u reserved " + "%u.%u", SCSB_TRACE_ARGS, __entry->rsv, __entry->rsv_holders, + __entry->res_items, __entry->res_vals, __entry->act_items, + __entry->act_vals, __entry->tri_holders, __entry->tri_writing, + __entry->tri_items, __entry->tri_vals) +); + +TRACE_EVENT(scoutfs_trans_acquired_hold, + TP_PROTO(struct super_block *sb, const struct scoutfs_item_count *cnt, + void *rsv, unsigned int rsv_holders, + struct scoutfs_item_count *res, + struct scoutfs_item_count *act, unsigned int tri_holders, + unsigned int tri_writing, unsigned int tri_items, + unsigned int tri_vals), + + TP_ARGS(sb, cnt, rsv, rsv_holders, res, act, tri_holders, tri_writing, + tri_items, tri_vals), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(int, cnt_items) + __field(int, cnt_vals) + __field(void *, rsv) + __field(unsigned int, rsv_holders) + __field(int, res_items) + __field(int, res_vals) + __field(int, act_items) + __field(int, act_vals) + __field(unsigned int, tri_holders) + __field(unsigned int, tri_writing) + __field(unsigned int, tri_items) + __field(unsigned int, tri_vals) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->cnt_items = cnt->items; + __entry->cnt_vals = cnt->vals; + __entry->rsv = rsv; + __entry->rsv_holders = rsv_holders; + __entry->res_items = res->items; + __entry->res_vals = res->vals; + __entry->act_items = act->items; + __entry->act_vals = act->vals; + __entry->tri_holders = tri_holders; + __entry->tri_writing = tri_writing; + __entry->tri_items = tri_items; + __entry->tri_vals = tri_vals; + ), + + TP_printk(SCSBF" cnt %u.%u, rsv %p holders %u reserved %u.%u " + "actual %d.%d, trans holders %u writing %u reserved " + "%u.%u", SCSB_TRACE_ARGS, __entry->cnt_items, + __entry->cnt_vals, __entry->rsv, __entry->rsv_holders, + __entry->res_items, __entry->res_vals, __entry->act_items, + __entry->act_vals, __entry->tri_holders, __entry->tri_writing, + __entry->tri_items, __entry->tri_vals) +); + +TRACE_EVENT(scoutfs_trans_track_item, + TP_PROTO(struct super_block *sb, int delta_items, int delta_vals, + int act_items, int act_vals, int res_items, int res_vals), + + TP_ARGS(sb, delta_items, delta_vals, act_items, act_vals, res_items, + res_vals), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(int, delta_items) + __field(int, delta_vals) + __field(int, act_items) + __field(int, act_vals) + __field(int, res_items) + __field(int, res_vals) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->delta_items = delta_items; + __entry->delta_vals = delta_vals; + __entry->act_items = act_items; + __entry->act_vals = act_vals; + __entry->res_items = res_items; + __entry->res_vals = res_vals; + ), + + TP_printk(SCSBF" delta_items %d delta_vals %d act_items %d act_vals %d res_items %d res_vals %d", + SCSB_TRACE_ARGS, __entry->delta_items, __entry->delta_vals, + __entry->act_items, __entry->act_vals, __entry->res_items, + __entry->res_vals) +); + +TRACE_EVENT(scoutfs_ioc_release, + TP_PROTO(struct super_block *sb, u64 ino, + struct scoutfs_ioctl_release *args), + + TP_ARGS(sb, ino, args), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(__u64, ino) + __field(__u64, block) + __field(__u64, count) + __field(__u64, vers) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->ino = ino; + __entry->block = args->block; + __entry->count = args->count; + __entry->vers = args->data_version; + ), + + TP_printk(SCSBF" ino %llu block %llu count %llu vers %llu", + SCSB_TRACE_ARGS, __entry->ino, __entry->block, + __entry->count, __entry->vers) +); + +DEFINE_EVENT(scoutfs_ino_ret_class, scoutfs_ioc_release_ret, + TP_PROTO(struct super_block *sb, u64 ino, int ret), + TP_ARGS(sb, ino, ret) +); + +TRACE_EVENT(scoutfs_ioc_stage, + TP_PROTO(struct super_block *sb, u64 ino, + struct scoutfs_ioctl_stage *args), + + TP_ARGS(sb, ino, args), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(__u64, ino) + __field(__u64, vers) + __field(__u64, offset) + __field(__s32, count) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->ino = ino; + __entry->vers = args->data_version; + __entry->offset = args->offset; + __entry->count = args->count; + ), + + TP_printk(SCSBF" ino %llu vers %llu offset %llu count %d", + SCSB_TRACE_ARGS, __entry->ino, __entry->vers, + __entry->offset, __entry->count) +); + +TRACE_EVENT(scoutfs_ioc_data_wait_err, + TP_PROTO(struct super_block *sb, + struct scoutfs_ioctl_data_wait_err *args), + + TP_ARGS(sb, args), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(__u64, ino) + __field(__u64, vers) + __field(__u64, offset) + __field(__u64, count) + __field(__u64, op) + __field(__s64, err) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->ino = args->ino; + __entry->vers = args->data_version; + __entry->offset = args->offset; + __entry->count = args->count; + __entry->op = args->op; + __entry->err = args->err; + ), + + TP_printk(SCSBF" ino %llu vers %llu offset %llu count %llu op %llx err %lld", + SCSB_TRACE_ARGS, __entry->ino, __entry->vers, + __entry->offset, __entry->count, __entry->op, __entry->err) +); + +DEFINE_EVENT(scoutfs_ino_ret_class, scoutfs_ioc_stage_ret, + TP_PROTO(struct super_block *sb, u64 ino, int ret), + TP_ARGS(sb, ino, ret) +); + +TRACE_EVENT(scoutfs_ioc_walk_inodes, + TP_PROTO(struct super_block *sb, struct scoutfs_ioctl_walk_inodes *walk), + + TP_ARGS(sb, walk), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(int, index) + __field(__u64, first_major) + __field(__u32, first_minor) + __field(__u64, first_ino) + __field(__u64, last_major) + __field(__u32, last_minor) + __field(__u64, last_ino) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->index = walk->index; + __entry->first_major = walk->first.major; + __entry->first_minor = walk->first.minor; + __entry->first_ino = walk->first.ino; + __entry->last_major = walk->last.major; + __entry->last_minor = walk->last.minor; + __entry->last_ino = walk->last.ino; + ), + + TP_printk(SCSBF" index %u first %llu.%u.%llu last %llu.%u.%llu", + SCSB_TRACE_ARGS, __entry->index, __entry->first_major, + __entry->first_minor, __entry->first_ino, __entry->last_major, + __entry->last_minor, __entry->last_ino) +); + +TRACE_EVENT(scoutfs_i_callback, + TP_PROTO(struct inode *inode), + + TP_ARGS(inode), + + TP_STRUCT__entry( + __field(struct inode *, inode) + ), + + TP_fast_assign( + __entry->inode = inode; + ), + + /* don't print fsid as we may not have our sb private available */ + TP_printk("freeing inode %p", __entry->inode) +); + +DECLARE_EVENT_CLASS(scoutfs_index_item_class, + TP_PROTO(struct super_block *sb, __u8 type, __u64 major, __u32 minor, + __u64 ino), + + TP_ARGS(sb, type, major, minor, ino), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(__u8, type) + __field(__u64, major) + __field(__u32, minor) + __field(__u64, ino) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->type = type; + __entry->major = major; + __entry->minor = minor; + __entry->ino = ino; + ), + + TP_printk(SCSBF" type %u major %llu minor %u ino %llu", + SCSB_TRACE_ARGS, __entry->type, __entry->major, + __entry->minor, __entry->ino) +); + +DEFINE_EVENT(scoutfs_index_item_class, scoutfs_create_index_item, + TP_PROTO(struct super_block *sb, __u8 type, __u64 major, __u32 minor, + __u64 ino), + TP_ARGS(sb, type, major, minor, ino) +); + +DEFINE_EVENT(scoutfs_index_item_class, scoutfs_delete_index_item, + TP_PROTO(struct super_block *sb, __u8 type, __u64 major, __u32 minor, + __u64 ino), + TP_ARGS(sb, type, major, minor, ino) +); + +TRACE_EVENT(scoutfs_alloc_ino, + TP_PROTO(struct super_block *sb, int ret, __u64 ino, __u64 next_ino, + __u64 next_nr), + + TP_ARGS(sb, ret, ino, next_ino, next_nr), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(int, ret) + __field(__u64, ino) + __field(__u64, next_ino) + __field(__u64, next_nr) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->ret = ret; + __entry->ino = ino; + __entry->next_ino = next_ino; + __entry->next_nr = next_nr; + ), + + TP_printk(SCSBF" ret %d ino %llu next_ino %llu next_nr %llu", + SCSB_TRACE_ARGS, __entry->ret, __entry->ino, + __entry->next_ino, __entry->next_nr) +); + +TRACE_EVENT(scoutfs_evict_inode, + TP_PROTO(struct super_block *sb, __u64 ino, unsigned int nlink, + unsigned int is_bad_ino), + + TP_ARGS(sb, ino, nlink, is_bad_ino), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(__u64, ino) + __field(unsigned int, nlink) + __field(unsigned int, is_bad_ino) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->ino = ino; + __entry->nlink = nlink; + __entry->is_bad_ino = is_bad_ino; + ), + + TP_printk(SCSBF" ino %llu nlink %u bad %d", SCSB_TRACE_ARGS, + __entry->ino, __entry->nlink, __entry->is_bad_ino) +); + +TRACE_EVENT(scoutfs_drop_inode, + TP_PROTO(struct super_block *sb, __u64 ino, unsigned int nlink, + unsigned int unhashed), + + TP_ARGS(sb, ino, nlink, unhashed), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(__u64, ino) + __field(unsigned int, nlink) + __field(unsigned int, unhashed) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->ino = ino; + __entry->nlink = nlink; + __entry->unhashed = unhashed; + ), + + TP_printk(SCSBF" ino %llu nlink %u unhashed %d", SCSB_TRACE_ARGS, + __entry->ino, __entry->nlink, __entry->unhashed) +); + +TRACE_EVENT(scoutfs_inode_walk_writeback, + TP_PROTO(struct super_block *sb, __u64 ino, int write, int ret), + + TP_ARGS(sb, ino, write, ret), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(__u64, ino) + __field(int, write) + __field(int, ret) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->ino = ino; + __entry->write = write; + __entry->ret = ret; + ), + + TP_printk(SCSBF" ino %llu write %d ret %d", SCSB_TRACE_ARGS, + __entry->ino, __entry->write, __entry->ret) +); + +DECLARE_EVENT_CLASS(scoutfs_lock_info_class, + TP_PROTO(struct super_block *sb, struct lock_info *linfo), + + TP_ARGS(sb, linfo), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(struct lock_info *, linfo) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->linfo = linfo; + ), + + TP_printk(SCSBF" linfo %p", SCSB_TRACE_ARGS, __entry->linfo) +); + +DEFINE_EVENT(scoutfs_lock_info_class, scoutfs_lock_setup, + TP_PROTO(struct super_block *sb, struct lock_info *linfo), + TP_ARGS(sb, linfo) +); + +DEFINE_EVENT(scoutfs_lock_info_class, scoutfs_lock_shutdown, + TP_PROTO(struct super_block *sb, struct lock_info *linfo), + TP_ARGS(sb, linfo) +); + +DEFINE_EVENT(scoutfs_lock_info_class, scoutfs_lock_destroy, + TP_PROTO(struct super_block *sb, struct lock_info *linfo), + TP_ARGS(sb, linfo) +); + +TRACE_EVENT(scoutfs_xattr_set, + TP_PROTO(struct super_block *sb, size_t name_len, const void *value, + size_t size, int flags), + + TP_ARGS(sb, name_len, value, size, flags), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(size_t, name_len) + __field(const void *, value) + __field(size_t, size) + __field(int, flags) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->name_len = name_len; + __entry->value = value; + __entry->size = size; + __entry->flags = flags; + ), + + TP_printk(SCSBF" name_len %zu value %p size %zu flags 0x%x", + SCSB_TRACE_ARGS, __entry->name_len, __entry->value, + __entry->size, __entry->flags) +); + +TRACE_EVENT(scoutfs_advance_dirty_super, + TP_PROTO(struct super_block *sb, __u64 seq), + + TP_ARGS(sb, seq), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(__u64, seq) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->seq = seq; + ), + + TP_printk(SCSBF" super seq now %llu", SCSB_TRACE_ARGS, __entry->seq) +); + +TRACE_EVENT(scoutfs_dir_add_next_linkref, + TP_PROTO(struct super_block *sb, __u64 ino, __u64 dir_ino, + __u64 dir_pos, int ret, __u64 found_dir_ino, + __u64 found_dir_pos, unsigned int name_len), + + TP_ARGS(sb, ino, dir_ino, dir_pos, ret, found_dir_pos, found_dir_ino, + name_len), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(__u64, ino) + __field(__u64, dir_ino) + __field(__u64, dir_pos) + __field(int, ret) + __field(__u64, found_dir_ino) + __field(__u64, found_dir_pos) + __field(unsigned int, name_len) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->ino = ino; + __entry->dir_ino = dir_ino; + __entry->dir_pos = dir_pos; + __entry->ret = ret; + __entry->found_dir_ino = dir_ino; + __entry->found_dir_pos = dir_pos; + __entry->name_len = name_len; + ), + + TP_printk(SCSBF" ino %llu dir_ino %llu dir_pos %llu ret %d found_dir_ino %llu found_dir_pos %llu name_len %u", + SCSB_TRACE_ARGS, __entry->ino, __entry->dir_pos, + __entry->dir_ino, __entry->ret, __entry->found_dir_pos, + __entry->found_dir_ino, __entry->name_len) +); + +TRACE_EVENT(scoutfs_write_begin, + TP_PROTO(struct super_block *sb, u64 ino, loff_t pos, unsigned len), + + TP_ARGS(sb, ino, pos, len), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(__u64, inode) + __field(__u64, pos) + __field(__u32, len) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->inode = ino; + __entry->pos = pos; + __entry->len = len; + ), + + TP_printk(SCSBF" ino %llu pos %llu len %u", SCSB_TRACE_ARGS, + __entry->inode, __entry->pos, __entry->len) +); + +TRACE_EVENT(scoutfs_write_end, + TP_PROTO(struct super_block *sb, u64 ino, unsigned long idx, u64 pos, + unsigned len, unsigned copied), + + TP_ARGS(sb, ino, idx, pos, len, copied), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(__u64, ino) + __field(unsigned long, idx) + __field(__u64, pos) + __field(__u32, len) + __field(__u32, copied) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->ino = ino; + __entry->idx = idx; + __entry->pos = pos; + __entry->len = len; + __entry->copied = copied; + ), + + TP_printk(SCSBF" ino %llu pgind %lu pos %llu len %u copied %d", + SCSB_TRACE_ARGS, __entry->ino, __entry->idx, __entry->pos, + __entry->len, __entry->copied) +); + +TRACE_EVENT(scoutfs_dirty_inode, + TP_PROTO(struct inode *inode), + + TP_ARGS(inode), + + TP_STRUCT__entry( + __field(__u64, ino) + __field(__u64, size) + ), + + TP_fast_assign( + __entry->ino = scoutfs_ino(inode); + __entry->size = inode->i_size; + ), + + TP_printk("ino %llu size %llu", + __entry->ino, __entry->size) +); + +TRACE_EVENT(scoutfs_update_inode, + TP_PROTO(struct inode *inode), + + TP_ARGS(inode), + + TP_STRUCT__entry( + __field(__u64, ino) + __field(__u64, size) + ), + + TP_fast_assign( + __entry->ino = scoutfs_ino(inode); + __entry->size = inode->i_size; + ), + + TP_printk("ino %llu size %llu", + __entry->ino, __entry->size) +); + +TRACE_EVENT(scoutfs_orphan_inode, + TP_PROTO(struct super_block *sb, struct inode *inode), + + TP_ARGS(sb, inode), + + TP_STRUCT__entry( + __field(dev_t, dev) + __field(__u64, ino) + ), + + TP_fast_assign( + __entry->dev = sb->s_dev; + __entry->ino = scoutfs_ino(inode); + ), + + TP_printk("dev %d,%d ino %llu", MAJOR(__entry->dev), + MINOR(__entry->dev), __entry->ino) +); + +TRACE_EVENT(scoutfs_delete_inode, + TP_PROTO(struct super_block *sb, u64 ino, umode_t mode, u64 size), + + TP_ARGS(sb, ino, mode, size), + + TP_STRUCT__entry( + __field(dev_t, dev) + __field(__u64, ino) + __field(umode_t, mode) + __field(__u64, size) + ), + + TP_fast_assign( + __entry->dev = sb->s_dev; + __entry->ino = ino; + __entry->mode = mode; + __entry->size = size; + ), + + TP_printk("dev %d,%d ino %llu, mode 0x%x size %llu", + MAJOR(__entry->dev), MINOR(__entry->dev), __entry->ino, + __entry->mode, __entry->size) +); + +TRACE_EVENT(scoutfs_scan_orphans, + TP_PROTO(struct super_block *sb), + + TP_ARGS(sb), + + TP_STRUCT__entry( + __field(dev_t, dev) + ), + + TP_fast_assign( + __entry->dev = sb->s_dev; + ), + + TP_printk("dev %d,%d", MAJOR(__entry->dev), MINOR(__entry->dev)) +); + +DECLARE_EVENT_CLASS(scoutfs_key_class, + TP_PROTO(struct super_block *sb, struct scoutfs_key *key), + TP_ARGS(sb, key), + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + sk_trace_define(key) + ), + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + sk_trace_assign(key, key); + ), + TP_printk(SCSBF" key "SK_FMT, SCSB_TRACE_ARGS, sk_trace_args(key)) +); + +DEFINE_EVENT(scoutfs_key_class, scoutfs_xattr_get_next_key, + TP_PROTO(struct super_block *sb, struct scoutfs_key *key), + TP_ARGS(sb, key) +); + +#define lock_mode(mode) \ + __print_symbolic(mode, \ + { SCOUTFS_LOCK_NULL, "NULL" }, \ + { SCOUTFS_LOCK_READ, "READ" }, \ + { SCOUTFS_LOCK_WRITE, "WRITE" }, \ + { SCOUTFS_LOCK_WRITE_ONLY, "WRITE_ONLY" }) + +DECLARE_EVENT_CLASS(scoutfs_lock_class, + TP_PROTO(struct super_block *sb, struct scoutfs_lock *lck), + TP_ARGS(sb, lck), + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + sk_trace_define(start) + sk_trace_define(end) + __field(u64, refresh_gen) + __field(unsigned char, request_pending) + __field(unsigned char, invalidate_pending) + __field(int, mode) + __field(unsigned int, waiters_cw) + __field(unsigned int, waiters_pr) + __field(unsigned int, waiters_ex) + __field(unsigned int, users_cw) + __field(unsigned int, users_pr) + __field(unsigned int, users_ex) + ), + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + sk_trace_assign(start, &lck->start); + sk_trace_assign(end, &lck->end); + __entry->refresh_gen = lck->refresh_gen; + __entry->request_pending = lck->request_pending; + __entry->invalidate_pending = lck->invalidate_pending; + __entry->mode = lck->mode; + __entry->waiters_pr = lck->waiters[SCOUTFS_LOCK_READ]; + __entry->waiters_ex = lck->waiters[SCOUTFS_LOCK_WRITE]; + __entry->waiters_cw = lck->waiters[SCOUTFS_LOCK_WRITE_ONLY]; + __entry->users_pr = lck->users[SCOUTFS_LOCK_READ]; + __entry->users_ex = lck->users[SCOUTFS_LOCK_WRITE]; + __entry->users_cw = lck->users[SCOUTFS_LOCK_WRITE_ONLY]; + ), + TP_printk(SCSBF" start "SK_FMT" end "SK_FMT" mode %u reqpnd %u invpnd %u rfrgen %llu waiters: pr %u ex %u cw %u users: pr %u ex %u cw %u", + SCSB_TRACE_ARGS, sk_trace_args(start), sk_trace_args(end), + __entry->mode, __entry->request_pending, + __entry->invalidate_pending, __entry->refresh_gen, + __entry->waiters_pr, __entry->waiters_ex, __entry->waiters_cw, + __entry->users_pr, __entry->users_ex, __entry->users_cw) +); +DEFINE_EVENT(scoutfs_lock_class, scoutfs_lock_invalidate, + TP_PROTO(struct super_block *sb, struct scoutfs_lock *lck), + TP_ARGS(sb, lck) +); +DEFINE_EVENT(scoutfs_lock_class, scoutfs_lock_free, + TP_PROTO(struct super_block *sb, struct scoutfs_lock *lck), + TP_ARGS(sb, lck) +); +DEFINE_EVENT(scoutfs_lock_class, scoutfs_lock_alloc, + TP_PROTO(struct super_block *sb, struct scoutfs_lock *lck), + TP_ARGS(sb, lck) +); +DEFINE_EVENT(scoutfs_lock_class, scoutfs_lock_grant_response, + TP_PROTO(struct super_block *sb, struct scoutfs_lock *lck), + TP_ARGS(sb, lck) +); +DEFINE_EVENT(scoutfs_lock_class, scoutfs_lock_granted, + TP_PROTO(struct super_block *sb, struct scoutfs_lock *lck), + TP_ARGS(sb, lck) +); +DEFINE_EVENT(scoutfs_lock_class, scoutfs_lock_invalidate_request, + TP_PROTO(struct super_block *sb, struct scoutfs_lock *lck), + TP_ARGS(sb, lck) +); +DEFINE_EVENT(scoutfs_lock_class, scoutfs_lock_invalidated, + TP_PROTO(struct super_block *sb, struct scoutfs_lock *lck), + TP_ARGS(sb, lck) +); +DEFINE_EVENT(scoutfs_lock_class, scoutfs_lock_locked, + TP_PROTO(struct super_block *sb, struct scoutfs_lock *lck), + TP_ARGS(sb, lck) +); +DEFINE_EVENT(scoutfs_lock_class, scoutfs_lock_wait, + TP_PROTO(struct super_block *sb, struct scoutfs_lock *lck), + TP_ARGS(sb, lck) +); +DEFINE_EVENT(scoutfs_lock_class, scoutfs_lock_unlock, + TP_PROTO(struct super_block *sb, struct scoutfs_lock *lck), + TP_ARGS(sb, lck) +); +DEFINE_EVENT(scoutfs_lock_class, scoutfs_lock_shrink, + TP_PROTO(struct super_block *sb, struct scoutfs_lock *lck), + TP_ARGS(sb, lck) +); + +DECLARE_EVENT_CLASS(scoutfs_net_class, + TP_PROTO(struct super_block *sb, struct sockaddr_in *name, + struct sockaddr_in *peer, struct scoutfs_net_header *nh), + TP_ARGS(sb, name, peer, nh), + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + si4_trace_define(name) + si4_trace_define(peer) + snh_trace_define(nh) + ), + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + si4_trace_assign(name, name); + si4_trace_assign(peer, peer); + snh_trace_assign(nh, nh); + ), + TP_printk(SCSBF" name "SI4_FMT" peer "SI4_FMT" nh "SNH_FMT, + SCSB_TRACE_ARGS, si4_trace_args(name), si4_trace_args(peer), + snh_trace_args(nh)) +); + +DEFINE_EVENT(scoutfs_net_class, scoutfs_net_send_message, + TP_PROTO(struct super_block *sb, struct sockaddr_in *name, + struct sockaddr_in *peer, struct scoutfs_net_header *nh), + TP_ARGS(sb, name, peer, nh) +); + +DEFINE_EVENT(scoutfs_net_class, scoutfs_net_recv_message, + TP_PROTO(struct super_block *sb, struct sockaddr_in *name, + struct sockaddr_in *peer, struct scoutfs_net_header *nh), + TP_ARGS(sb, name, peer, nh) +); + +#define conn_flag_entry(which) \ + CONN_FL_##which, __stringify(which) + +#define print_conn_flags(flags) __print_flags(flags, "|", \ + { conn_flag_entry(valid_greeting) }, \ + { conn_flag_entry(established) }, \ + { conn_flag_entry(shutting_down) }, \ + { conn_flag_entry(saw_greeting) }, \ + { conn_flag_entry(saw_farewell) }, \ + { conn_flag_entry(reconn_wait) }, \ + { conn_flag_entry(reconn_freeing) }) + +/* + * This is called from alloc and free when the caller only has safe + * access to the struct itself, be very careful not to follow any + * indirection out of the storage for the conn struct. + */ +DECLARE_EVENT_CLASS(scoutfs_net_conn_class, + TP_PROTO(struct scoutfs_net_connection *conn), + TP_ARGS(conn), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(unsigned long, flags) + __field(unsigned long, reconn_deadline) + __field(unsigned long, connect_timeout_ms) + __field(void *, sock) + __field(__u64, c_rid) + __field(__u64, greeting_id) + si4_trace_define(sockname) + si4_trace_define(peername) + __field(unsigned char, e_accepted_head) + __field(void *, listening_conn) + __field(unsigned char, e_accepted_list) + __field(__u64, next_send_seq) + __field(__u64, next_send_id) + __field(unsigned char, e_send_queue) + __field(unsigned char, e_resend_queue) + __field(__u64, recv_seq) + ), + TP_fast_assign( + SCSB_TRACE_ASSIGN(conn->sb); + __entry->flags = conn->flags; + __entry->reconn_deadline = conn->reconn_deadline; + __entry->connect_timeout_ms = conn->connect_timeout_ms; + __entry->sock = conn->sock; + __entry->c_rid = conn->rid; + __entry->greeting_id = conn->greeting_id; + si4_trace_assign(sockname, &conn->sockname); + si4_trace_assign(peername, &conn->peername); + __entry->e_accepted_head = !!list_empty(&conn->accepted_head); + __entry->listening_conn = conn->listening_conn; + __entry->e_accepted_list = !!list_empty(&conn->accepted_list); + __entry->next_send_seq = conn->next_send_seq; + __entry->next_send_id = conn->next_send_id; + __entry->e_send_queue = !!list_empty(&conn->send_queue); + __entry->e_resend_queue = !!list_empty(&conn->resend_queue); + __entry->recv_seq = atomic64_read(&conn->recv_seq); + ), + TP_printk(SCSBF" flags %s rc_dl %lu cto %lu sk %p rid %llu grid %llu sn "SI4_FMT" pn "SI4_FMT" eah %u lc %p eal %u nss %llu nsi %llu esq %u erq %u rs %llu", + SCSB_TRACE_ARGS, + print_conn_flags(__entry->flags), + __entry->reconn_deadline, + __entry->connect_timeout_ms, + __entry->sock, + __entry->c_rid, + __entry->greeting_id, + si4_trace_args(sockname), + si4_trace_args(peername), + __entry->e_accepted_head, + __entry->listening_conn, + __entry->e_accepted_list, + __entry->next_send_seq, + __entry->next_send_id, + __entry->e_send_queue, + __entry->e_resend_queue, + __entry->recv_seq) +); +DEFINE_EVENT(scoutfs_net_conn_class, scoutfs_conn_alloc, + TP_PROTO(struct scoutfs_net_connection *conn), + TP_ARGS(conn) +); +DEFINE_EVENT(scoutfs_net_conn_class, scoutfs_conn_connect_start, + TP_PROTO(struct scoutfs_net_connection *conn), + TP_ARGS(conn) +); +DEFINE_EVENT(scoutfs_net_conn_class, scoutfs_conn_connect_result, + TP_PROTO(struct scoutfs_net_connection *conn), + TP_ARGS(conn) +); +DEFINE_EVENT(scoutfs_net_conn_class, scoutfs_conn_connect_complete, + TP_PROTO(struct scoutfs_net_connection *conn), + TP_ARGS(conn) +); +DEFINE_EVENT(scoutfs_net_conn_class, scoutfs_conn_accept, + TP_PROTO(struct scoutfs_net_connection *conn), + TP_ARGS(conn) +); +DEFINE_EVENT(scoutfs_net_conn_class, scoutfs_conn_reconn_migrate, + TP_PROTO(struct scoutfs_net_connection *conn), + TP_ARGS(conn) +); +DEFINE_EVENT(scoutfs_net_conn_class, scoutfs_conn_shutdown_queued, + TP_PROTO(struct scoutfs_net_connection *conn), + TP_ARGS(conn) +); +DEFINE_EVENT(scoutfs_net_conn_class, scoutfs_conn_shutdown_start, + TP_PROTO(struct scoutfs_net_connection *conn), + TP_ARGS(conn) +); +DEFINE_EVENT(scoutfs_net_conn_class, scoutfs_conn_shutdown_complete, + TP_PROTO(struct scoutfs_net_connection *conn), + TP_ARGS(conn) +); +DEFINE_EVENT(scoutfs_net_conn_class, scoutfs_conn_destroy_start, + TP_PROTO(struct scoutfs_net_connection *conn), + TP_ARGS(conn) +); +DEFINE_EVENT(scoutfs_net_conn_class, scoutfs_conn_destroy_free, + TP_PROTO(struct scoutfs_net_connection *conn), + TP_ARGS(conn) +); + +DECLARE_EVENT_CLASS(scoutfs_work_class, + TP_PROTO(struct super_block *sb, u64 data, int ret), + TP_ARGS(sb, data, ret), + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(__u64, data) + __field(int, ret) + ), + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->data = data; + __entry->ret = ret; + ), + TP_printk(SCSBF" data %llu ret %d", + SCSB_TRACE_ARGS, __entry->data, __entry->ret) +); +DEFINE_EVENT(scoutfs_work_class, scoutfs_server_commit_work_enter, + TP_PROTO(struct super_block *sb, u64 data, int ret), + TP_ARGS(sb, data, ret) +); +DEFINE_EVENT(scoutfs_work_class, scoutfs_server_commit_work_exit, + TP_PROTO(struct super_block *sb, u64 data, int ret), + TP_ARGS(sb, data, ret) +); +DEFINE_EVENT(scoutfs_work_class, scoutfs_net_proc_work_enter, + TP_PROTO(struct super_block *sb, u64 data, int ret), + TP_ARGS(sb, data, ret) +); +DEFINE_EVENT(scoutfs_work_class, scoutfs_net_proc_work_exit, + TP_PROTO(struct super_block *sb, u64 data, int ret), + TP_ARGS(sb, data, ret) +); +DEFINE_EVENT(scoutfs_work_class, scoutfs_net_listen_work_enter, + TP_PROTO(struct super_block *sb, u64 data, int ret), + TP_ARGS(sb, data, ret) +); +DEFINE_EVENT(scoutfs_work_class, scoutfs_net_listen_work_exit, + TP_PROTO(struct super_block *sb, u64 data, int ret), + TP_ARGS(sb, data, ret) +); +DEFINE_EVENT(scoutfs_work_class, scoutfs_net_connect_work_enter, + TP_PROTO(struct super_block *sb, u64 data, int ret), + TP_ARGS(sb, data, ret) +); +DEFINE_EVENT(scoutfs_work_class, scoutfs_net_connect_work_exit, + TP_PROTO(struct super_block *sb, u64 data, int ret), + TP_ARGS(sb, data, ret) +); +DEFINE_EVENT(scoutfs_work_class, scoutfs_net_shutdown_work_enter, + TP_PROTO(struct super_block *sb, u64 data, int ret), + TP_ARGS(sb, data, ret) +); +DEFINE_EVENT(scoutfs_work_class, scoutfs_net_shutdown_work_exit, + TP_PROTO(struct super_block *sb, u64 data, int ret), + TP_ARGS(sb, data, ret) +); +DEFINE_EVENT(scoutfs_work_class, scoutfs_net_destroy_work_enter, + TP_PROTO(struct super_block *sb, u64 data, int ret), + TP_ARGS(sb, data, ret) +); +DEFINE_EVENT(scoutfs_work_class, scoutfs_net_destroy_work_exit, + TP_PROTO(struct super_block *sb, u64 data, int ret), + TP_ARGS(sb, data, ret) +); +DEFINE_EVENT(scoutfs_work_class, scoutfs_net_reconn_free_work_enter, + TP_PROTO(struct super_block *sb, u64 data, int ret), + TP_ARGS(sb, data, ret) +); +DEFINE_EVENT(scoutfs_work_class, scoutfs_net_reconn_free_work_exit, + TP_PROTO(struct super_block *sb, u64 data, int ret), + TP_ARGS(sb, data, ret) +); +DEFINE_EVENT(scoutfs_work_class, scoutfs_net_send_work_enter, + TP_PROTO(struct super_block *sb, u64 data, int ret), + TP_ARGS(sb, data, ret) +); +DEFINE_EVENT(scoutfs_work_class, scoutfs_net_send_work_exit, + TP_PROTO(struct super_block *sb, u64 data, int ret), + TP_ARGS(sb, data, ret) +); +DEFINE_EVENT(scoutfs_work_class, scoutfs_net_recv_work_enter, + TP_PROTO(struct super_block *sb, u64 data, int ret), + TP_ARGS(sb, data, ret) +); +DEFINE_EVENT(scoutfs_work_class, scoutfs_net_recv_work_exit, + TP_PROTO(struct super_block *sb, u64 data, int ret), + TP_ARGS(sb, data, ret) +); +DEFINE_EVENT(scoutfs_work_class, scoutfs_server_work_enter, + TP_PROTO(struct super_block *sb, u64 data, int ret), + TP_ARGS(sb, data, ret) +); +DEFINE_EVENT(scoutfs_work_class, scoutfs_server_work_exit, + TP_PROTO(struct super_block *sb, u64 data, int ret), + TP_ARGS(sb, data, ret) +); +DEFINE_EVENT(scoutfs_work_class, scoutfs_server_workqueue_destroy, + TP_PROTO(struct super_block *sb, u64 data, int ret), + TP_ARGS(sb, data, ret) +); +DEFINE_EVENT(scoutfs_work_class, scoutfs_data_return_server_extents_enter, + TP_PROTO(struct super_block *sb, u64 data, int ret), + TP_ARGS(sb, data, ret) +); +DEFINE_EVENT(scoutfs_work_class, scoutfs_data_return_server_extents_exit, + TP_PROTO(struct super_block *sb, u64 data, int ret), + TP_ARGS(sb, data, ret) +); + +DECLARE_EVENT_CLASS(scoutfs_shrink_exit_class, + TP_PROTO(struct super_block *sb, unsigned long nr_to_scan, int ret), + TP_ARGS(sb, nr_to_scan, ret), + TP_STRUCT__entry( + __field(void *, sb) + __field(unsigned long, nr_to_scan) + __field(int, ret) + ), + TP_fast_assign( + __entry->sb = sb; + __entry->nr_to_scan = nr_to_scan; + __entry->ret = ret; + ), + TP_printk("sb %p nr_to_scan %lu ret %d", + __entry->sb, __entry->nr_to_scan, __entry->ret) +); + +DEFINE_EVENT(scoutfs_shrink_exit_class, scoutfs_lock_shrink_exit, + TP_PROTO(struct super_block *sb, unsigned long nr_to_scan, int ret), + TP_ARGS(sb, nr_to_scan, ret) +); + +TRACE_EVENT(scoutfs_rename, + TP_PROTO(struct super_block *sb, struct inode *old_dir, + struct dentry *old_dentry, struct inode *new_dir, + struct dentry *new_dentry), + + TP_ARGS(sb, old_dir, old_dentry, new_dir, new_dentry), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(__u64, old_dir_ino) + __string(old_name, old_dentry->d_name.name) + __field(__u64, new_dir_ino) + __string(new_name, new_dentry->d_name.name) + __field(__u64, new_inode_ino) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->old_dir_ino = scoutfs_ino(old_dir); + __assign_str(old_name, old_dentry->d_name.name) + __entry->new_dir_ino = scoutfs_ino(new_dir); + __assign_str(new_name, new_dentry->d_name.name) + __entry->new_inode_ino = new_dentry->d_inode ? + scoutfs_ino(new_dentry->d_inode) : 0; + ), + + TP_printk(SCSBF" old_dir_ino %llu old_name %s new_dir_ino %llu new_name %s new_inode_ino %llu", + SCSB_TRACE_ARGS, __entry->old_dir_ino, __get_str(old_name), + __entry->new_dir_ino, __get_str(new_name), + __entry->new_inode_ino) +); + +TRACE_EVENT(scoutfs_d_revalidate, + TP_PROTO(struct super_block *sb, + struct dentry *dentry, int flags, struct dentry *parent, + bool is_covered, int ret), + + TP_ARGS(sb, dentry, flags, parent, is_covered, ret), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __string(name, dentry->d_name.name) + __field(__u64, ino) + __field(__u64, parent_ino) + __field(int, flags) + __field(int, is_root) + __field(int, is_covered) + __field(int, ret) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __assign_str(name, dentry->d_name.name) + __entry->ino = dentry->d_inode ? + scoutfs_ino(dentry->d_inode) : 0; + __entry->parent_ino = parent->d_inode ? + scoutfs_ino(parent->d_inode) : 0; + __entry->flags = flags; + __entry->is_root = IS_ROOT(dentry); + __entry->is_covered = is_covered; + __entry->ret = ret; + ), + + TP_printk(SCSBF" name %s ino %llu parent_ino %llu flags 0x%x s_root %u is_covered %u ret %d", + SCSB_TRACE_ARGS, __get_str(name), __entry->ino, + __entry->parent_ino, __entry->flags, + __entry->is_root, + __entry->is_covered, + __entry->ret) +); + +DECLARE_EVENT_CLASS(scoutfs_super_lifecycle_class, + TP_PROTO(struct super_block *sb), + TP_ARGS(sb), + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(void *, sb) + __field(void *, sbi) + __field(void *, s_root) + ), + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->sb = sb; + __entry->sbi = SCOUTFS_SB(sb); + __entry->s_root = sb->s_root; + ), + TP_printk(SCSBF" sb %p sbi %p s_root %p", + SCSB_TRACE_ARGS, __entry->sb, __entry->sbi, __entry->s_root) +); + +DEFINE_EVENT(scoutfs_super_lifecycle_class, scoutfs_fill_super, + TP_PROTO(struct super_block *sb), + TP_ARGS(sb) +); + +DEFINE_EVENT(scoutfs_super_lifecycle_class, scoutfs_put_super, + TP_PROTO(struct super_block *sb), + TP_ARGS(sb) +); + +DEFINE_EVENT(scoutfs_super_lifecycle_class, scoutfs_kill_sb, + TP_PROTO(struct super_block *sb), + TP_ARGS(sb) +); + +DECLARE_EVENT_CLASS(scoutfs_fileid_class, + TP_PROTO(struct super_block *sb, int fh_type, struct scoutfs_fid *fid), + TP_ARGS(sb, fh_type, fid), + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(int, fh_type) + __field(u64, ino) + __field(u64, parent_ino) + ), + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->fh_type = fh_type; + __entry->ino = le64_to_cpu(fid->ino); + __entry->parent_ino = fh_type == FILEID_SCOUTFS_WITH_PARENT ? + le64_to_cpu(fid->parent_ino) : 0ULL; + ), + TP_printk(SCSBF" type %d ino %llu parent %llu", + SCSB_TRACE_ARGS, __entry->fh_type, __entry->ino, + __entry->parent_ino) +); + +DEFINE_EVENT(scoutfs_fileid_class, scoutfs_encode_fh, + TP_PROTO(struct super_block *sb, int fh_type, struct scoutfs_fid *fid), + TP_ARGS(sb, fh_type, fid) +); + +DEFINE_EVENT(scoutfs_fileid_class, scoutfs_fh_to_dentry, + TP_PROTO(struct super_block *sb, int fh_type, struct scoutfs_fid *fid), + TP_ARGS(sb, fh_type, fid) +); + +DEFINE_EVENT(scoutfs_fileid_class, scoutfs_fh_to_parent, + TP_PROTO(struct super_block *sb, int fh_type, struct scoutfs_fid *fid), + TP_ARGS(sb, fh_type, fid) +); + +TRACE_EVENT(scoutfs_get_parent, + TP_PROTO(struct super_block *sb, struct inode *inode, u64 parent), + + TP_ARGS(sb, inode, parent), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(__u64, ino) + __field(__u64, parent) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->ino = scoutfs_ino(inode); + __entry->parent = parent; + ), + + TP_printk(SCSBF" child %llu parent %llu", + SCSB_TRACE_ARGS, __entry->ino, __entry->parent) +); + +TRACE_EVENT(scoutfs_get_name, + TP_PROTO(struct super_block *sb, struct inode *parent, + struct inode *child, char *name), + + TP_ARGS(sb, parent, child, name), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(__u64, parent_ino) + __field(__u64, child_ino) + __string(name, name) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->parent_ino = scoutfs_ino(parent); + __entry->child_ino = scoutfs_ino(child); + __assign_str(name, name); + ), + + TP_printk(SCSBF" parent %llu child %llu name: %s", + SCSB_TRACE_ARGS, __entry->parent_ino, __entry->child_ino, + __get_str(name)) +); + +TRACE_EVENT(scoutfs_btree_read_error, + TP_PROTO(struct super_block *sb, struct scoutfs_btree_ref *ref), + + TP_ARGS(sb, ref), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(__u64, blkno) + __field(__u64, seq) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->blkno = le64_to_cpu(ref->blkno); + __entry->seq = le64_to_cpu(ref->seq); + ), + + TP_printk(SCSBF" blkno %llu seq %llu", + SCSB_TRACE_ARGS, __entry->blkno, __entry->seq) +); + +TRACE_EVENT(scoutfs_btree_dirty_block, + TP_PROTO(struct super_block *sb, u64 blkno, u64 seq, + u64 bt_blkno, u64 bt_seq), + + TP_ARGS(sb, blkno, seq, bt_blkno, bt_seq), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(__u64, blkno) + __field(__u64, seq) + __field(__u64, bt_blkno) + __field(__u64, bt_seq) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->blkno = blkno; + __entry->seq = seq; + __entry->bt_blkno = bt_blkno; + __entry->bt_seq = bt_seq; + ), + + TP_printk(SCSBF" blkno %llu seq %llu bt_blkno %llu bt_seq %llu", + SCSB_TRACE_ARGS, __entry->blkno, __entry->seq, + __entry->bt_blkno, __entry->bt_seq) +); + +TRACE_EVENT(scoutfs_btree_walk, + TP_PROTO(struct super_block *sb, struct scoutfs_btree_root *root, + struct scoutfs_key *key, int flags, int level, + struct scoutfs_btree_ref *ref), + + TP_ARGS(sb, root, key, flags, level, ref), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(__u64, root_blkno) + __field(__u64, root_seq) + __field(__u8, root_height) + sk_trace_define(key) + __field(int, flags) + __field(int, level) + __field(__u64, ref_blkno) + __field(__u64, ref_seq) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->root_blkno = le64_to_cpu(root->ref.blkno); + __entry->root_seq = le64_to_cpu(root->ref.seq); + __entry->root_height = root->height; + sk_trace_assign(key, key); + __entry->flags = flags; + __entry->level = level; + __entry->ref_blkno = le64_to_cpu(ref->blkno); + __entry->ref_seq = le64_to_cpu(ref->seq); + ), + + TP_printk(SCSBF" root blkno %llu seq %llu height %u key "SK_FMT" flags 0x%x level %d ref blkno %llu seq %llu", + SCSB_TRACE_ARGS, __entry->root_blkno, __entry->root_seq, + __entry->root_height, sk_trace_args(key), __entry->flags, + __entry->level, __entry->ref_blkno, __entry->ref_seq) +); + +TRACE_EVENT(scoutfs_online_offline_blocks, + TP_PROTO(struct inode *inode, s64 on_delta, s64 off_delta, + u64 on_now, u64 off_now), + + TP_ARGS(inode, on_delta, off_delta, on_now, off_now), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(__s64, on_delta) + __field(__s64, off_delta) + __field(__u64, on_now) + __field(__u64, off_now) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(inode->i_sb); + __entry->on_delta = on_delta; + __entry->off_delta = off_delta; + __entry->on_now = on_now; + __entry->off_now = off_now; + ), + + TP_printk(SCSBF" on_delta %lld off_delta %lld on_now %llu off_now %llu ", + SCSB_TRACE_ARGS, __entry->on_delta, __entry->off_delta, + __entry->on_now, __entry->off_now) +); + +DECLARE_EVENT_CLASS(scoutfs_server_client_count_class, + TP_PROTO(struct super_block *sb, u64 rid, unsigned long nr_clients), + + TP_ARGS(sb, rid, nr_clients), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(__s64, c_rid) + __field(unsigned long, nr_clients) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->c_rid = rid; + __entry->nr_clients = nr_clients; + ), + + TP_printk(SCSBF" rid %016llx nr_clients %lu", + SCSB_TRACE_ARGS, __entry->c_rid, __entry->nr_clients) +); +DEFINE_EVENT(scoutfs_server_client_count_class, scoutfs_server_client_up, + TP_PROTO(struct super_block *sb, u64 rid, unsigned long nr_clients), + TP_ARGS(sb, rid, nr_clients) +); +DEFINE_EVENT(scoutfs_server_client_count_class, scoutfs_server_client_down, + TP_PROTO(struct super_block *sb, u64 rid, unsigned long nr_clients), + TP_ARGS(sb, rid, nr_clients) +); + +#define slt_symbolic(mode) \ + __print_symbolic(mode, \ + { SLT_CLIENT, "client" }, \ + { SLT_SERVER, "server" }, \ + { SLT_GRANT, "grant" }, \ + { SLT_INVALIDATE, "invalidate" }, \ + { SLT_REQUEST, "request" }, \ + { SLT_RESPONSE, "response" }) + +TRACE_EVENT(scoutfs_lock_message, + TP_PROTO(struct super_block *sb, int who, int what, int dir, + u64 rid, u64 net_id, struct scoutfs_net_lock *nl), + + TP_ARGS(sb, who, what, dir, rid, net_id, nl), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(int, who) + __field(int, what) + __field(int, dir) + __field(__u64, m_rid) + __field(__u64, net_id) + sk_trace_define(key) + __field(__u8, old_mode) + __field(__u8, new_mode) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->who = who; + __entry->what = what; + __entry->dir = dir; + __entry->m_rid = rid; + __entry->net_id = net_id; + sk_trace_assign(key, &nl->key); + __entry->old_mode = nl->old_mode; + __entry->new_mode = nl->new_mode; + ), + + TP_printk(SCSBF" %s %s %s rid %016llx net_id %llu key "SK_FMT" old_mode %u new_mode %u", + SCSB_TRACE_ARGS, slt_symbolic(__entry->who), + slt_symbolic(__entry->what), slt_symbolic(__entry->dir), + __entry->m_rid, __entry->net_id, sk_trace_args(key), + __entry->old_mode, __entry->new_mode) +); + + +TRACE_EVENT(scoutfs_quorum_election, + TP_PROTO(struct super_block *sb, u64 prev_term), + + TP_ARGS(sb, prev_term), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(__u64, prev_term) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->prev_term = prev_term; + ), + + TP_printk(SCSBF" prev_term %llu", + SCSB_TRACE_ARGS, __entry->prev_term) +); + +TRACE_EVENT(scoutfs_quorum_election_ret, + TP_PROTO(struct super_block *sb, int ret, u64 elected_term), + + TP_ARGS(sb, ret, elected_term), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(int, ret) + __field(__u64, elected_term) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->ret = ret; + __entry->elected_term = elected_term; + ), + + TP_printk(SCSBF" ret %d elected_term %llu", + SCSB_TRACE_ARGS, __entry->ret, __entry->elected_term) +); + +TRACE_EVENT(scoutfs_quorum_election_vote, + TP_PROTO(struct super_block *sb, int role, u64 term, u64 vote_for_rid, + int votes, int log_cycles, int quorum_count), + + TP_ARGS(sb, role, term, vote_for_rid, votes, log_cycles, quorum_count), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(int, role) + __field(__u64, term) + __field(__u64, vote_for_rid) + __field(int, votes) + __field(int, log_cycles) + __field(int, quorum_count) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->role = role; + __entry->term = term; + __entry->vote_for_rid = vote_for_rid; + __entry->votes = votes; + __entry->log_cycles = log_cycles; + __entry->quorum_count = quorum_count; + ), + + TP_printk(SCSBF" role %d term %llu vote_for_rid %016llx votes %d log_cycles %d quorum_count %d", + SCSB_TRACE_ARGS, __entry->role, __entry->term, + __entry->vote_for_rid, __entry->votes, __entry->log_cycles, + __entry->quorum_count) +); + +DECLARE_EVENT_CLASS(scoutfs_quorum_block_class, + TP_PROTO(struct super_block *sb, struct scoutfs_quorum_block *blk), + + TP_ARGS(sb, blk), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(__u64, blkno) + __field(__u64, term) + __field(__u64, write_nr) + __field(__u64, voter_rid) + __field(__u64, vote_for_rid) + __field(__u32, crc) + __field(__u8, log_nr) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->blkno = le64_to_cpu(blk->blkno); + __entry->term = le64_to_cpu(blk->term); + __entry->write_nr = le64_to_cpu(blk->write_nr); + __entry->voter_rid = le64_to_cpu(blk->voter_rid); + __entry->vote_for_rid = le64_to_cpu(blk->vote_for_rid); + __entry->crc = le32_to_cpu(blk->crc); + __entry->log_nr = blk->log_nr; + ), + + TP_printk(SCSBF" blkno %llu term %llu write_nr %llu voter_rid %016llx vote_for_rid %016llx crc 0x%08x log_nr %u", + SCSB_TRACE_ARGS, __entry->blkno, __entry->term, + __entry->write_nr, __entry->voter_rid, __entry->vote_for_rid, + __entry->crc, __entry->log_nr) +); +DEFINE_EVENT(scoutfs_quorum_block_class, scoutfs_quorum_read_block, + TP_PROTO(struct super_block *sb, struct scoutfs_quorum_block *blk), + TP_ARGS(sb, blk) +); +DEFINE_EVENT(scoutfs_quorum_block_class, scoutfs_quorum_write_block, + TP_PROTO(struct super_block *sb, struct scoutfs_quorum_block *blk), + TP_ARGS(sb, blk) +); + +/* + * We can emit trace events to make it easier to synchronize the + * monotonic clocks in trace logs between nodes. By looking at the send + * and recv times of many messages flowing between nodes we can get + * surprisingly good estimates of the clock offset between them. + */ +DECLARE_EVENT_CLASS(scoutfs_clock_sync_class, + TP_PROTO(__le64 clock_sync_id), + TP_ARGS(clock_sync_id), + TP_STRUCT__entry( + __field(__u64, clock_sync_id) + ), + TP_fast_assign( + __entry->clock_sync_id = le64_to_cpu(clock_sync_id); + ), + TP_printk("clock_sync_id %016llx", __entry->clock_sync_id) +); +DEFINE_EVENT(scoutfs_clock_sync_class, scoutfs_send_clock_sync, + TP_PROTO(__le64 clock_sync_id), + TP_ARGS(clock_sync_id) +); +DEFINE_EVENT(scoutfs_clock_sync_class, scoutfs_recv_clock_sync, + TP_PROTO(__le64 clock_sync_id), + TP_ARGS(clock_sync_id) +); + +TRACE_EVENT(scoutfs_trans_seq_advance, + TP_PROTO(struct super_block *sb, u64 rid, u64 prev_seq, + u64 next_seq), + + TP_ARGS(sb, rid, prev_seq, next_seq), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(__u64, s_rid) + __field(__u64, prev_seq) + __field(__u64, next_seq) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->s_rid = rid; + __entry->prev_seq = prev_seq; + __entry->next_seq = next_seq; + ), + + TP_printk(SCSBF" rid %016llx prev_seq %llu next_seq %llu", + SCSB_TRACE_ARGS, __entry->s_rid, __entry->prev_seq, + __entry->next_seq) +); + +TRACE_EVENT(scoutfs_trans_seq_farewell, + TP_PROTO(struct super_block *sb, u64 rid, u64 trans_seq), + + TP_ARGS(sb, rid, trans_seq), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(__u64, s_rid) + __field(__u64, trans_seq) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->s_rid = rid; + __entry->trans_seq = trans_seq; + ), + + TP_printk(SCSBF" rid %016llx trans_seq %llu", + SCSB_TRACE_ARGS, __entry->s_rid, __entry->trans_seq) +); + +TRACE_EVENT(scoutfs_trans_seq_last, + TP_PROTO(struct super_block *sb, u64 rid, u64 trans_seq), + + TP_ARGS(sb, rid, trans_seq), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(__u64, s_rid) + __field(__u64, trans_seq) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->s_rid = rid; + __entry->trans_seq = trans_seq; + ), + + TP_printk(SCSBF" rid %016llx trans_seq %llu", + SCSB_TRACE_ARGS, __entry->s_rid, __entry->trans_seq) +); + +DECLARE_EVENT_CLASS(scoutfs_forest_bloom_class, + TP_PROTO(struct super_block *sb, struct scoutfs_key *key, + u64 rid, u64 nr, u64 blkno, u64 seq, unsigned int count), + TP_ARGS(sb, key, rid, nr, blkno, seq, count), + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + sk_trace_define(key) + __field(__u64, b_rid) + __field(__u64, nr) + __field(__u64, blkno) + __field(__u64, seq) + __field(unsigned int, count) + ), + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + sk_trace_assign(key, key); + __entry->b_rid = rid; + __entry->nr = nr; + __entry->blkno = blkno; + __entry->seq = seq; + __entry->count = count; + ), + TP_printk(SCSBF" key "SK_FMT" rid %016llx nr %llu blkno %llu seq %llx count %u", + SCSB_TRACE_ARGS, sk_trace_args(key), __entry->b_rid, + __entry->nr, __entry->blkno, __entry->seq, __entry->count) +); +DEFINE_EVENT(scoutfs_forest_bloom_class, scoutfs_forest_bloom_set, + TP_PROTO(struct super_block *sb, struct scoutfs_key *key, + u64 rid, u64 nr, u64 blkno, u64 seq, unsigned int count), + TP_ARGS(sb, key, rid, nr, blkno, seq, count) +); +DEFINE_EVENT(scoutfs_forest_bloom_class, scoutfs_forest_bloom_search, + TP_PROTO(struct super_block *sb, struct scoutfs_key *key, + u64 rid, u64 nr, u64 blkno, u64 seq, unsigned int count), + TP_ARGS(sb, key, rid, nr, blkno, seq, count) +); + +TRACE_EVENT(scoutfs_forest_prepare_commit, + TP_PROTO(struct super_block *sb, struct scoutfs_btree_ref *item_ref, + struct scoutfs_btree_ref *bloom_ref), + TP_ARGS(sb, item_ref, bloom_ref), + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(__u64, item_blkno) + __field(__u64, item_seq) + __field(__u64, bloom_blkno) + __field(__u64, bloom_seq) + ), + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->item_blkno = le64_to_cpu(item_ref->blkno); + __entry->item_seq = le64_to_cpu(item_ref->seq); + __entry->bloom_blkno = le64_to_cpu(bloom_ref->blkno); + __entry->bloom_seq = le64_to_cpu(bloom_ref->seq); + ), + TP_printk(SCSBF" item blkno %llu seq %llu bloom blkno %llu seq %llu", + SCSB_TRACE_ARGS, __entry->item_blkno, __entry->item_seq, + __entry->bloom_blkno, __entry->bloom_seq) +); + +TRACE_EVENT(scoutfs_forest_using_roots, + TP_PROTO(struct super_block *sb, struct scoutfs_btree_root *fs_root, + struct scoutfs_btree_root *logs_root), + TP_ARGS(sb, fs_root, logs_root), + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(__u64, fs_blkno) + __field(__u64, fs_seq) + __field(__u64, logs_blkno) + __field(__u64, logs_seq) + ), + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->fs_blkno = le64_to_cpu(fs_root->ref.blkno); + __entry->fs_seq = le64_to_cpu(fs_root->ref.seq); + __entry->logs_blkno = le64_to_cpu(logs_root->ref.blkno); + __entry->logs_seq = le64_to_cpu(logs_root->ref.seq); + ), + TP_printk(SCSBF" fs blkno %llu seq %llu logs blkno %llu seq %llu", + SCSB_TRACE_ARGS, __entry->fs_blkno, __entry->fs_seq, + __entry->logs_blkno, __entry->logs_seq) +); + +TRACE_EVENT(scoutfs_forest_init_our_log, + TP_PROTO(struct super_block *sb, u64 rid, u64 nr, u64 blkno, u64 seq), + TP_ARGS(sb, rid, nr, blkno, seq), + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(__u64, b_rid) + __field(__u64, nr) + __field(__u64, blkno) + __field(__u64, seq) + ), + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->b_rid = rid; + __entry->nr = nr; + __entry->blkno = blkno; + __entry->seq = seq; + ), + TP_printk(SCSBF" rid %016llx nr %llu blkno %llu seq %llx", + SCSB_TRACE_ARGS, __entry->b_rid, __entry->nr, + __entry->blkno, __entry->seq) +); + +DECLARE_EVENT_CLASS(scoutfs_block_class, + TP_PROTO(struct super_block *sb, void *bp, u64 blkno, + int refcount, int io_count, unsigned long bits, u64 lru_moved), + TP_ARGS(sb, bp, blkno, refcount, io_count, bits, lru_moved), + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(void *, bp) + __field(__u64, blkno) + __field(int, refcount) + __field(int, io_count) + __field(unsigned long, bits) + __field(__u64, lru_moved) + ), + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->bp = bp; + __entry->blkno = blkno; + __entry->refcount = refcount; + __entry->io_count = io_count; + __entry->bits = bits; + __entry->lru_moved = lru_moved; + ), + TP_printk(SCSBF" bp %p blkno %llu refcount %d io_count %d bits 0x%lx lru_moved %llu", + SCSB_TRACE_ARGS, __entry->bp, __entry->blkno, + __entry->refcount, __entry->io_count, __entry->bits, + __entry->lru_moved) +); +DEFINE_EVENT(scoutfs_block_class, scoutfs_block_allocate, + TP_PROTO(struct super_block *sb, void *bp, u64 blkno, + int refcount, int io_count, unsigned long bits, u64 lru_moved), + TP_ARGS(sb, bp, blkno, refcount, io_count, bits, lru_moved) +); +DEFINE_EVENT(scoutfs_block_class, scoutfs_block_free, + TP_PROTO(struct super_block *sb, void *bp, u64 blkno, + int refcount, int io_count, unsigned long bits, u64 lru_moved), + TP_ARGS(sb, bp, blkno, refcount, io_count, bits, lru_moved) +); +DEFINE_EVENT(scoutfs_block_class, scoutfs_block_insert, + TP_PROTO(struct super_block *sb, void *bp, u64 blkno, + int refcount, int io_count, unsigned long bits, u64 lru_moved), + TP_ARGS(sb, bp, blkno, refcount, io_count, bits, lru_moved) +); +DEFINE_EVENT(scoutfs_block_class, scoutfs_block_end_io, + TP_PROTO(struct super_block *sb, void *bp, u64 blkno, + int refcount, int io_count, unsigned long bits, u64 lru_moved), + TP_ARGS(sb, bp, blkno, refcount, io_count, bits, lru_moved) +); +DEFINE_EVENT(scoutfs_block_class, scoutfs_block_submit, + TP_PROTO(struct super_block *sb, void *bp, u64 blkno, + int refcount, int io_count, unsigned long bits, u64 lru_moved), + TP_ARGS(sb, bp, blkno, refcount, io_count, bits, lru_moved) +); +DEFINE_EVENT(scoutfs_block_class, scoutfs_block_invalidate, + TP_PROTO(struct super_block *sb, void *bp, u64 blkno, + int refcount, int io_count, unsigned long bits, u64 lru_moved), + TP_ARGS(sb, bp, blkno, refcount, io_count, bits, lru_moved) +); +DEFINE_EVENT(scoutfs_block_class, scoutfs_block_mark_dirty, + TP_PROTO(struct super_block *sb, void *bp, u64 blkno, + int refcount, int io_count, unsigned long bits, u64 lru_moved), + TP_ARGS(sb, bp, blkno, refcount, io_count, bits, lru_moved) +); +DEFINE_EVENT(scoutfs_block_class, scoutfs_block_forget, + TP_PROTO(struct super_block *sb, void *bp, u64 blkno, + int refcount, int io_count, unsigned long bits, u64 lru_moved), + TP_ARGS(sb, bp, blkno, refcount, io_count, bits, lru_moved) +); +DEFINE_EVENT(scoutfs_block_class, scoutfs_block_shrink, + TP_PROTO(struct super_block *sb, void *bp, u64 blkno, + int refcount, int io_count, unsigned long bits, u64 lru_moved), + TP_ARGS(sb, bp, blkno, refcount, io_count, bits, lru_moved) +); + +DECLARE_EVENT_CLASS(scoutfs_ext_next_class, + TP_PROTO(struct super_block *sb, u64 start, u64 len, + struct scoutfs_extent *ext, int ret), + + TP_ARGS(sb, start, len, ext, ret), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(__u64, start) + __field(__u64, len) + STE_FIELDS(ext) + __field(int, ret) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->start = start; + __entry->len = len; + STE_ASSIGN(ext, ext) + __entry->ret = ret; + ), + + TP_printk(SCSBF" start %llu len %llu ext "STE_FMT" ret %d", + SCSB_TRACE_ARGS, __entry->start, __entry->len, + STE_ENTRY_ARGS(ext), __entry->ret) +); + +DEFINE_EVENT(scoutfs_ext_next_class, scoutfs_ext_op_next, + TP_PROTO(struct super_block *sb, u64 start, u64 len, + struct scoutfs_extent *ext, int ret), + TP_ARGS(sb, start, len, ext, ret) +); +DEFINE_EVENT(scoutfs_ext_next_class, scoutfs_ext_next, + TP_PROTO(struct super_block *sb, u64 start, u64 len, + struct scoutfs_extent *ext, int ret), + TP_ARGS(sb, start, len, ext, ret) +); + +DECLARE_EVENT_CLASS(scoutfs_ext_typical_class, + TP_PROTO(struct super_block *sb, u64 start, u64 len, u64 map, u8 flags, + int ret), + + TP_ARGS(sb, start, len, map, flags, ret), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(__u64, start) + __field(__u64, len) + __field(__u64, map) + __field(__u8, flags) + __field(int, ret) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->start = start; + __entry->len = len; + __entry->map = map; + __entry->flags = flags; + __entry->ret = ret; + ), + + TP_printk(SCSBF" start %llu len %llu map %llu flags %u ret %d", + SCSB_TRACE_ARGS, __entry->start, __entry->len, __entry->map, + __entry->flags, __entry->ret) +); + +DEFINE_EVENT(scoutfs_ext_typical_class, scoutfs_ext_op_insert, + TP_PROTO(struct super_block *sb, u64 start, u64 len, u64 map, u8 flags, + int ret), + TP_ARGS(sb, start, len, map, flags, ret) +); +DEFINE_EVENT(scoutfs_ext_typical_class, scoutfs_ext_insert, + TP_PROTO(struct super_block *sb, u64 start, u64 len, u64 map, u8 flags, + int ret), + TP_ARGS(sb, start, len, map, flags, ret) +); +DEFINE_EVENT(scoutfs_ext_typical_class, scoutfs_ext_op_remove, + TP_PROTO(struct super_block *sb, u64 start, u64 len, u64 map, u8 flags, + int ret), + TP_ARGS(sb, start, len, map, flags, ret) +); +DEFINE_EVENT(scoutfs_ext_typical_class, scoutfs_ext_remove, + TP_PROTO(struct super_block *sb, u64 start, u64 len, u64 map, u8 flags, + int ret), + TP_ARGS(sb, start, len, map, flags, ret) +); +DEFINE_EVENT(scoutfs_ext_typical_class, scoutfs_ext_set, + TP_PROTO(struct super_block *sb, u64 start, u64 len, u64 map, u8 flags, + int ret), + TP_ARGS(sb, start, len, map, flags, ret) +); + +TRACE_EVENT(scoutfs_ext_alloc, + TP_PROTO(struct super_block *sb, u64 start, u64 len, u64 count, + struct scoutfs_extent *ext, int ret), + + TP_ARGS(sb, start, len, count, ext, ret), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(__u64, start) + __field(__u64, len) + __field(__u64, count) + STE_FIELDS(ext) + __field(int, ret) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->start = start; + __entry->len = len; + __entry->count = count; + STE_ASSIGN(ext, ext) + __entry->ret = ret; + ), + + TP_printk(SCSBF" start %llu len %llu count %llu ext "STE_FMT" ret %d", + SCSB_TRACE_ARGS, __entry->start, __entry->len, __entry->count, + STE_ENTRY_ARGS(ext), __entry->ret) +); + +TRACE_EVENT(scoutfs_alloc_alloc_meta, + TP_PROTO(struct super_block *sb, u64 blkno, int ret), + + TP_ARGS(sb, blkno, ret), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(__u64, blkno) + __field(int, ret) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->blkno = blkno; + __entry->ret = ret; + ), + + TP_printk(SCSBF" blkno %llu ret %d", + SCSB_TRACE_ARGS, __entry->blkno, __entry->ret) +); + +TRACE_EVENT(scoutfs_alloc_free_meta, + TP_PROTO(struct super_block *sb, u64 blkno, int ret), + + TP_ARGS(sb, blkno, ret), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(__u64, blkno) + __field(int, ret) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->blkno = blkno; + __entry->ret = ret; + ), + + TP_printk(SCSBF" blkno %llu ret %d", + SCSB_TRACE_ARGS, __entry->blkno, __entry->ret) +); + +TRACE_EVENT(scoutfs_alloc_alloc_data, + TP_PROTO(struct super_block *sb, u64 req, u64 blkno, u64 count, + int ret), + + TP_ARGS(sb, req, blkno, count, ret), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(__u64, req) + __field(__u64, blkno) + __field(__u64, count) + __field(int, ret) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->req = req; + __entry->blkno = blkno; + __entry->count = count; + __entry->ret = ret; + ), + + TP_printk(SCSBF" req %llu blkno %llu count %llu ret %d", + SCSB_TRACE_ARGS, __entry->req, __entry->blkno, + __entry->count, __entry->ret) +); + +TRACE_EVENT(scoutfs_alloc_free_data, + TP_PROTO(struct super_block *sb, u64 blkno, u64 count, int ret), + + TP_ARGS(sb, blkno, count, ret), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(__u64, blkno) + __field(__u64, count) + __field(int, ret) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->blkno = blkno; + __entry->count = count; + __entry->ret = ret; + ), + + TP_printk(SCSBF" blkno %llu count %llu ret %d", + SCSB_TRACE_ARGS, __entry->blkno, __entry->count, + __entry->ret) +); + +TRACE_EVENT(scoutfs_alloc_move, + TP_PROTO(struct super_block *sb, u64 total, u64 moved, int ret), + + TP_ARGS(sb, total, moved, ret), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(__u64, total) + __field(__u64, moved) + __field(int, ret) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->total = total; + __entry->moved = moved; + __entry->ret = ret; + ), + + TP_printk(SCSBF" total %llu moved %llu ret %d", + SCSB_TRACE_ARGS, __entry->total, __entry->moved, + __entry->ret) +); + +#endif /* _TRACE_SCOUTFS_H */ + +/* This part must be outside protection */ +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH . +#define TRACE_INCLUDE_FILE scoutfs_trace +#include diff --git a/kmod/src/server.c b/kmod/src/server.c new file mode 100644 index 00000000..57a7e8d5 --- /dev/null +++ b/kmod/src/server.c @@ -0,0 +1,1768 @@ +/* + * Copyright (C) 2018 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "format.h" +#include "counters.h" +#include "inode.h" +#include "block.h" +#include "btree.h" +#include "scoutfs_trace.h" +#include "msg.h" +#include "server.h" +#include "net.h" +#include "lock_server.h" +#include "endian_swap.h" +#include "quorum.h" +#include "trans.h" +#include "srch.h" +#include "alloc.h" +#include "forest.h" + +/* + * Every active mount can act as the server that listens on a net + * connection and accepts connections from all the other mounts acting + * as clients. + * + * The server is started by the mount that is elected leader by quorum. + * If it sees errors it shuts down the server in the hopes that another + * mount will become the leader and have less trouble. + */ + +struct server_info { + struct super_block *sb; + spinlock_t lock; + wait_queue_head_t waitq; + + struct workqueue_struct *wq; + struct work_struct work; + int err; + bool shutting_down; + struct completion start_comp; + struct sockaddr_in listen_sin; + u64 term; + struct scoutfs_net_connection *conn; + + /* request processing coordinates shared commits */ + struct rw_semaphore commit_rwsem; + struct llist_head commit_waiters; + struct work_struct commit_work; + + /* server tracks seq use */ + struct rw_semaphore seq_rwsem; + + struct list_head clients; + unsigned long nr_clients; + + /* track clients waiting in unmmount for farewell response */ + struct mutex farewell_mutex; + struct list_head farewell_requests; + struct work_struct farewell_work; + + struct mutex alloc_mutex; + /* swap between two fs meta roots to increase time to reuse */ + struct scoutfs_alloc_root *meta_avail; + struct scoutfs_alloc_root *meta_freed; + /* server's meta allocators alternate between persistent heads */ + struct scoutfs_alloc alloc; + int other_ind; + struct scoutfs_alloc_list_head *other_avail; + struct scoutfs_alloc_list_head *other_freed; + struct scoutfs_block_writer wri; + + struct mutex logs_mutex; + struct mutex srch_mutex; + + /* stable versions stored from commits, given in locks and rpcs */ + seqcount_t roots_seqcount; + struct scoutfs_net_roots roots; +}; + +#define DECLARE_SERVER_INFO(sb, name) \ + struct server_info *name = SCOUTFS_SB(sb)->server_info + +/* + * The server tracks each connected client. + */ +struct server_client_info { + u64 rid; + struct list_head head; +}; + +struct commit_waiter { + struct completion comp; + struct llist_node node; + int ret; +}; + +static void stop_server(struct server_info *server) +{ + /* wait_event/wake_up provide barriers */ + server->shutting_down = true; + wake_up(&server->waitq); +} + +/* + * Hold the shared rwsem that lets multiple holders modify blocks in the + * current commit and prevents the commit worker from acquiring the + * exclusive write lock to write the commit. + * + * This is exported for server components isolated in their own files + * (lock_server) and which are not called directly by the server core + * (async timeout work). + */ +int scoutfs_server_hold_commit(struct super_block *sb) +{ + DECLARE_SERVER_INFO(sb, server); + + scoutfs_inc_counter(sb, server_commit_hold); + + down_read(&server->commit_rwsem); + + return 0; +} + +/* + * This is called while holding the commit and returns once the commit + * is successfully written. Many holders can all wait for all holders + * to drain before their shared commit is applied and they're all woken. + * + * It's important to realize that our commit_waiter list node might be + * serviced by a currently executing commit work that is blocked waiting + * for the holders to release the commit_rwsem. This caller can return + * from wait_for_commit() while another future commit_work is still + * queued. + * + * This could queue delayed work but we're first trying to have batching + * work by having concurrent modification line up behind a commit in + * flight. Once the commit finishes it'll unlock and hopefully everyone + * will race to make their changes and they'll all be applied by the + * next commit after that. + */ +int scoutfs_server_apply_commit(struct super_block *sb, int err) +{ + DECLARE_SERVER_INFO(sb, server); + struct commit_waiter cw; + + if (err == 0) { + cw.ret = 0; + init_completion(&cw.comp); + llist_add(&cw.node, &server->commit_waiters); + scoutfs_inc_counter(sb, server_commit_queue); + queue_work(server->wq, &server->commit_work); + } + + up_read(&server->commit_rwsem); + + if (err == 0) { + wait_for_completion(&cw.comp); + err = cw.ret; + } + + return err; +} + +void scoutfs_server_get_roots(struct super_block *sb, + struct scoutfs_net_roots *roots) +{ + DECLARE_SERVER_INFO(sb, server); + unsigned int seq; + + do { + seq = read_seqcount_begin(&server->roots_seqcount); + *roots = server->roots; + } while (read_seqcount_retry(&server->roots_seqcount, seq)); +} + +static void set_roots(struct server_info *server, + struct scoutfs_btree_root *fs_root, + struct scoutfs_btree_root *logs_root, + struct scoutfs_btree_root *srch_root) +{ + preempt_disable(); + write_seqcount_begin(&server->roots_seqcount); + server->roots.fs_root = *fs_root; + server->roots.logs_root = *logs_root; + server->roots.srch_root = *srch_root; + write_seqcount_end(&server->roots_seqcount); + preempt_enable(); +} + +/* + * Concurrent request processing dirties blocks in a commit and makes + * the modifications persistent before replying. We'd like to batch + * these commits as much as is reasonable so that we don't degrade to a + * few IO round trips per request. + * + * Getting that batching right is bound up in the concurrency of request + * processing so a clear way to implement the batched commits is to + * implement commits with a single pending work func like the + * processing. + * + * Processing paths acquire the rwsem for reading while they're making + * multiple dependent changes. When they're done and want it persistent + * they add themselves to the list of waiters and queue the commit work. + * This work runs, acquires the lock to exclude other writers, and + * performs the commit. Readers can run concurrently with these + * commits. + */ +static void scoutfs_server_commit_func(struct work_struct *work) +{ + struct server_info *server = container_of(work, struct server_info, + commit_work); + struct super_block *sb = server->sb; + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + struct commit_waiter *cw; + struct commit_waiter *pos; + struct llist_node *node; + int ret; + + trace_scoutfs_server_commit_work_enter(sb, 0, 0); + scoutfs_inc_counter(sb, server_commit_worker); + + down_write(&server->commit_rwsem); + + /* make sure next avail has sufficient blocks */ + ret = scoutfs_alloc_fill_list(sb, &server->alloc, &server->wri, + server->other_avail, + server->meta_avail, + SCOUTFS_SERVER_META_FILL_LO, + SCOUTFS_SERVER_META_FILL_TARGET); + if (ret) { + scoutfs_err(sb, "server error refilling avail: %d", ret); + goto out; + } + + /* merge freed blocks into extents, might be partial */ + ret = scoutfs_alloc_empty_list(sb, &server->alloc, &server->wri, + server->meta_freed, + server->other_freed); + if (ret) { + scoutfs_err(sb, "server error emptying freed: %d", ret); + goto out; + } + + ret = scoutfs_alloc_prepare_commit(sb, &server->alloc, &server->wri); + if (ret < 0) { + scoutfs_err(sb, "server error prepare alloc commit: %d", ret); + goto out; + } + + ret = scoutfs_block_writer_write(sb, &server->wri); + if (ret) { + scoutfs_err(sb, "server error writing btree blocks: %d", ret); + goto out; + } + + super->server_meta_avail[server->other_ind ^ 1] = server->alloc.avail; + super->server_meta_freed[server->other_ind ^ 1] = server->alloc.freed; + + ret = scoutfs_write_super(sb, super); + if (ret) { + scoutfs_err(sb, "server error writing super block: %d", ret); + goto out; + } + + set_roots(server, &super->fs_root, &super->logs_root, + &super->srch_root); + + /* swizzle the active and idle server alloc/freed heads */ + server->other_ind ^= 1; + server->alloc.avail = super->server_meta_avail[server->other_ind ^ 1]; + server->alloc.freed = super->server_meta_freed[server->other_ind ^ 1]; + server->other_avail = &super->server_meta_avail[server->other_ind]; + server->other_freed = &super->server_meta_freed[server->other_ind]; + + /* swap avail/free if avail gets low and freed is high */ + if (le64_to_cpu(server->meta_avail->total_len) <= + SCOUTFS_SERVER_META_ALLOC_MIN && + le64_to_cpu(server->meta_freed->total_len) > + SCOUTFS_SERVER_META_ALLOC_MIN) + swap(server->meta_avail, server->meta_freed); + + ret = 0; +out: + node = llist_del_all(&server->commit_waiters); + + /* waiters always wait on completion, cw could be free after complete */ + llist_for_each_entry_safe(cw, pos, node, node) { + cw->ret = ret; + complete(&cw->comp); + } + + up_write(&server->commit_rwsem); + trace_scoutfs_server_commit_work_exit(sb, 0, ret); +} + +static int server_alloc_inodes(struct super_block *sb, + struct scoutfs_net_connection *conn, + u8 cmd, u64 id, void *arg, u16 arg_len) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->super; + struct scoutfs_net_inode_alloc ial = { 0, }; + __le64 lecount; + u64 ino; + u64 nr; + int ret; + + if (arg_len != sizeof(lecount)) { + ret = -EINVAL; + goto out; + } + + memcpy(&lecount, arg, arg_len); + + ret = scoutfs_server_hold_commit(sb); + if (ret) + goto out; + + spin_lock(&sbi->next_ino_lock); + ino = le64_to_cpu(super->next_ino); + nr = min(le64_to_cpu(lecount), U64_MAX - ino); + le64_add_cpu(&super->next_ino, nr); + spin_unlock(&sbi->next_ino_lock); + + ret = scoutfs_server_apply_commit(sb, ret); + if (ret == 0) { + ial.ino = cpu_to_le64(ino); + ial.nr = cpu_to_le64(nr); + } +out: + return scoutfs_net_response(sb, conn, cmd, id, ret, &ial, sizeof(ial)); +} + +/* + * Refill the destination root if it's fallen below the lo threshold by + * moving from the src root to bring it up to the target. + */ +static int alloc_move_refill(struct super_block *sb, + struct scoutfs_alloc_root *dst, + struct scoutfs_alloc_root *src, u64 lo, u64 target) +{ + DECLARE_SERVER_INFO(sb, server); + + if (le64_to_cpu(dst->total_len) >= lo) + return 0; + + return scoutfs_alloc_move(sb, &server->alloc, &server->wri, dst, src, + min(target - le64_to_cpu(dst->total_len), + le64_to_cpu(src->total_len))); +} + +static int alloc_move_empty(struct super_block *sb, + struct scoutfs_alloc_root *dst, + struct scoutfs_alloc_root *src) +{ + DECLARE_SERVER_INFO(sb, server); + + return scoutfs_alloc_move(sb, &server->alloc, &server->wri, + dst, src, le64_to_cpu(src->total_len)); +} + +/* + * Give the client roots to all the trees that they'll use to build + * their transaction. + * + * We make sure that their alloc trees have sufficient blocks to + * allocate metadata and data for the transaction. We merge their freed + * trees back into the core allocators. They're were committed with the + * previous transaction so they're stable and can now be reused, even by + * the server in this commit. + */ +static int server_get_log_trees(struct super_block *sb, + struct scoutfs_net_connection *conn, + u8 cmd, u64 id, void *arg, u16 arg_len) +{ + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + u64 rid = scoutfs_net_client_rid(conn); + DECLARE_SERVER_INFO(sb, server); + SCOUTFS_BTREE_ITEM_REF(iref); + struct scoutfs_log_trees lt; + struct scoutfs_key key; + int ret; + + if (arg_len != 0) { + ret = -EINVAL; + goto out; + } + + ret = scoutfs_server_hold_commit(sb); + if (ret) + goto out; + + mutex_lock(&server->logs_mutex); + + scoutfs_key_init_log_trees(&key, rid, U64_MAX); + + ret = scoutfs_btree_prev(sb, &super->logs_root, &key, &iref); + if (ret < 0 && ret != -ENOENT) + goto unlock; + if (ret == 0) { + if (iref.val_len == sizeof(struct scoutfs_log_trees)) { + key = *iref.key; + memcpy(<, iref.val, iref.val_len); + if (le64_to_cpu(key.sklt_rid) != rid) + ret = -ENOENT; + } else { + ret = -EIO; + } + scoutfs_btree_put_iref(&iref); + if (ret == -EIO) + goto unlock; + } + + /* initialize new roots if we don't have any */ + if (ret == -ENOENT) { + key.sklt_rid = cpu_to_le64(rid); + key.sklt_nr = cpu_to_le64(1); + memset(<, 0, sizeof(lt)); + lt.rid = key.sklt_rid; + lt.nr = key.sklt_nr; + } + + /* return freed to server for emptying, refill avail */ + mutex_lock(&server->alloc_mutex); + ret = scoutfs_alloc_splice_list(sb, &server->alloc, &server->wri, + server->other_freed, + <.meta_freed) ?: + alloc_move_empty(sb, &super->data_alloc, <.data_freed) ?: + scoutfs_alloc_fill_list(sb, &server->alloc, &server->wri, + <.meta_avail, server->meta_avail, + SCOUTFS_SERVER_META_FILL_LO, + SCOUTFS_SERVER_META_FILL_TARGET) ?: + alloc_move_refill(sb, <.data_avail, &super->data_alloc, + SCOUTFS_SERVER_DATA_FILL_LO, + SCOUTFS_SERVER_DATA_FILL_TARGET); + mutex_unlock(&server->alloc_mutex); + if (ret < 0) + goto unlock; + + /* update client's log tree's item */ + ret = scoutfs_btree_force(sb, &server->alloc, &server->wri, + &super->logs_root, &key, <, sizeof(lt)); +unlock: + mutex_unlock(&server->logs_mutex); + + ret = scoutfs_server_apply_commit(sb, ret); +out: + WARN_ON_ONCE(ret < 0); + return scoutfs_net_response(sb, conn, cmd, id, ret, <, sizeof(lt)); +} + +/* + * The client is sending the roots of all the btree blocks that they + * wrote to their free space for their transaction. Make it persistent + * by referencing the roots from their log item in the logs root and + * committing. + */ +static int server_commit_log_trees(struct super_block *sb, + struct scoutfs_net_connection *conn, + u8 cmd, u64 id, void *arg, u16 arg_len) +{ + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + DECLARE_SERVER_INFO(sb, server); + SCOUTFS_BTREE_ITEM_REF(iref); + struct scoutfs_log_trees lt; + struct scoutfs_key key; + int ret; + + if (arg_len != sizeof(struct scoutfs_log_trees)) { + ret = -EINVAL; + goto out; + } + + /* don't modify the caller's log_trees */ + memcpy(<, arg, sizeof(struct scoutfs_log_trees)); + + ret = scoutfs_server_hold_commit(sb); + if (ret < 0) { + scoutfs_err(sb, "server error preparing commit: %d", ret); + goto out; + } + + mutex_lock(&server->logs_mutex); + + /* find the client's existing item */ + scoutfs_key_init_log_trees(&key, le64_to_cpu(lt.rid), + le64_to_cpu(lt.nr)); + ret = scoutfs_btree_lookup(sb, &super->logs_root, &key, &iref); + if (ret < 0) { + scoutfs_err(sb, "server error finding client logs: %d", ret); + goto unlock; + } + if (ret == 0) + scoutfs_btree_put_iref(&iref); + + /* try to rotate the srch log when big enough */ + mutex_lock(&server->srch_mutex); + ret = scoutfs_srch_rotate_log(sb, &server->alloc, &server->wri, + &super->srch_root, <.srch_file); + mutex_unlock(&server->srch_mutex); + if (ret < 0) { + scoutfs_err(sb, "server error, rotating srch log: %d", ret); + goto unlock; + } + + ret = scoutfs_btree_update(sb, &server->alloc, &server->wri, + &super->logs_root, &key, <, sizeof(lt)); + if (ret < 0) + scoutfs_err(sb, "server error updating client logs: %d", ret); + +unlock: + mutex_unlock(&server->logs_mutex); + + ret = scoutfs_server_apply_commit(sb, ret); + if (ret < 0) + scoutfs_err(sb, "server error commiting client logs: %d", ret); +out: + WARN_ON_ONCE(ret < 0); + return scoutfs_net_response(sb, conn, cmd, id, ret, NULL, 0); +} + +/* + * Give the client the most recent version of the fs btrees that are + * visible in persistent storage. We don't want to accidentally give + * them our in-memory dirty version. This can be racing with commits. + */ +static int server_get_roots(struct super_block *sb, + struct scoutfs_net_connection *conn, + u8 cmd, u64 id, void *arg, u16 arg_len) +{ + struct scoutfs_net_roots roots; + int ret; + + if (arg_len != 0) { + memset(&roots, 0, sizeof(roots)); + ret = -EINVAL; + } else { + scoutfs_server_get_roots(sb, &roots); + ret = 0; + } + + return scoutfs_net_response(sb, conn, cmd, id, 0, + &roots, sizeof(roots)); +} + +/* + * A client is being evicted so we want to reclaim resources from their + * log tree items. The item trees and bloom refs stay around to be read + * and eventually merged and we reclaim all the allocator items. + * + * The caller holds the commit rwsem which means we do all this work in + * one server commit. We'll need to keep the total amount of blocks in + * trees in check. + * + * By the time we're evicting a client they've either synced their data + * or have been forcefully removed. The free blocks in the allocator + * roots are stable and can be merged back into allocator items for use + * without risking overwriting stable data. + * + * We can return an error without fully reclaiming all the log item's + * referenced data. + */ +static int reclaim_log_trees(struct super_block *sb, u64 rid) +{ + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + DECLARE_SERVER_INFO(sb, server); + SCOUTFS_BTREE_ITEM_REF(iref); + struct scoutfs_log_trees lt; + struct scoutfs_key key; + int ret; + int err; + + mutex_lock(&server->logs_mutex); + + /* find the client's existing item */ + scoutfs_key_init_log_trees(&key, rid, 0); + ret = scoutfs_btree_next(sb, &super->logs_root, &key, &iref); + if (ret == 0) { + if (iref.val_len == sizeof(struct scoutfs_log_trees)) { + key = *iref.key; + memcpy(<, iref.val, iref.val_len); + if (le64_to_cpu(key.sklt_rid) != rid) + ret = -ENOENT; + } else { + ret = -EIO; + } + scoutfs_btree_put_iref(&iref); + } + if (ret < 0) { + if (ret == -ENOENT) + ret = 0; + goto out; + } + + /* + * All of these can return errors after having modified the + * allocator trees. We have to try and update the roots in the + * log item. + */ + mutex_lock(&server->alloc_mutex); + ret = scoutfs_alloc_splice_list(sb, &server->alloc, &server->wri, + server->other_freed, + <.meta_freed) ?: + scoutfs_alloc_splice_list(sb, &server->alloc, &server->wri, + server->other_freed, + <.meta_avail) ?: + alloc_move_empty(sb, &super->data_alloc, <.data_avail) ?: + alloc_move_empty(sb, &super->data_alloc, <.data_freed); + mutex_unlock(&server->alloc_mutex); + + err = scoutfs_btree_update(sb, &server->alloc, &server->wri, + &super->logs_root, &key, <, sizeof(lt)); + BUG_ON(err != 0); /* alloc and log item roots out of sync */ + +out: + mutex_unlock(&server->logs_mutex); + + return ret; +} + +static void init_trans_seq_key(struct scoutfs_key *key, u64 seq, u64 rid) +{ + *key = (struct scoutfs_key) { + .sk_zone = SCOUTFS_TRANS_SEQ_ZONE, + .skts_trans_seq = cpu_to_le64(seq), + .skts_rid = cpu_to_le64(rid), + }; +} + +/* + * Give the client the next sequence number for their transaction. They + * provide their previous transaction sequence number that they've + * committed. + * + * We track the sequence numbers of transactions that clients have open. + * This limits the transaction sequence numbers that can be returned in + * the index of inodes by meta and data transaction numbers. We + * communicate the largest possible sequence number to clients via an + * rpc. + * + * The transaction sequence tracking is stored in a btree so it is + * shared across servers. Final entries are removed when processing a + * client's farewell or when it's removed. + */ +static int server_advance_seq(struct super_block *sb, + struct scoutfs_net_connection *conn, + u8 cmd, u64 id, void *arg, u16 arg_len) +{ + DECLARE_SERVER_INFO(sb, server); + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->super; + __le64 their_seq; + __le64 next_seq; + u64 rid = scoutfs_net_client_rid(conn); + struct scoutfs_key key; + int ret; + + if (arg_len != sizeof(__le64)) { + ret = -EINVAL; + goto out; + } + memcpy(&their_seq, arg, sizeof(their_seq)); + + ret = scoutfs_server_hold_commit(sb); + if (ret) + goto out; + + down_write(&server->seq_rwsem); + + if (their_seq != 0) { + init_trans_seq_key(&key, le64_to_cpu(their_seq), rid); + ret = scoutfs_btree_delete(sb, &server->alloc, &server->wri, + &super->trans_seqs, &key); + if (ret < 0 && ret != -ENOENT) + goto unlock; + } + + next_seq = super->next_trans_seq; + le64_add_cpu(&super->next_trans_seq, 1); + + trace_scoutfs_trans_seq_advance(sb, rid, le64_to_cpu(their_seq), + le64_to_cpu(next_seq)); + + init_trans_seq_key(&key, le64_to_cpu(next_seq), rid); + ret = scoutfs_btree_insert(sb, &server->alloc, &server->wri, + &super->trans_seqs, &key, NULL, 0); +unlock: + up_write(&server->seq_rwsem); + ret = scoutfs_server_apply_commit(sb, ret); + +out: + return scoutfs_net_response(sb, conn, cmd, id, ret, + &next_seq, sizeof(next_seq)); +} + +/* + * Remove any transaction sequences owned by the client. They must have + * committed any final transaction by the time they get here via sending + * their farewell message. This can be called multiple times as the + * client's farewell is retransmitted so it's OK to not find any + * entries. This is called with the server commit rwsem held. + */ +static int remove_trans_seq(struct super_block *sb, u64 rid) +{ + DECLARE_SERVER_INFO(sb, server); + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->super; + SCOUTFS_BTREE_ITEM_REF(iref); + struct scoutfs_key key; + int ret = 0; + + down_write(&server->seq_rwsem); + + init_trans_seq_key(&key, 0, 0); + + for (;;) { + ret = scoutfs_btree_next(sb, &super->trans_seqs, &key, &iref); + if (ret < 0) { + if (ret == -ENOENT) + ret = 0; + break; + } + + key = *iref.key; + scoutfs_btree_put_iref(&iref); + + if (le64_to_cpu(key.skts_rid) == rid) { + trace_scoutfs_trans_seq_farewell(sb, rid, + le64_to_cpu(key.skts_trans_seq)); + ret = scoutfs_btree_delete(sb, &server->alloc, + &server->wri, + &super->trans_seqs, &key); + break; + } + + scoutfs_key_inc(&key); + } + + up_write(&server->seq_rwsem); + + return ret; +} + +/* + * Give the calling client the last valid trans_seq that it can return + * in results from the indices of trans seqs to inodes. These indices + * promise to only advance so we can't return results past those that + * are still outstanding and not yet visible in the indices. If there + * are no outstanding transactions (what? how?) we give them the max + * possible sequence. + */ +static int server_get_last_seq(struct super_block *sb, + struct scoutfs_net_connection *conn, + u8 cmd, u64 id, void *arg, u16 arg_len) +{ + DECLARE_SERVER_INFO(sb, server); + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->super; + SCOUTFS_BTREE_ITEM_REF(iref); + u64 rid = scoutfs_net_client_rid(conn); + struct scoutfs_key key; + __le64 last_seq = 0; + int ret; + + if (arg_len != 0) { + ret = -EINVAL; + goto out; + } + + down_read(&server->seq_rwsem); + + init_trans_seq_key(&key, 0, 0); + ret = scoutfs_btree_next(sb, &super->trans_seqs, &key, &iref); + if (ret == 0) { + key = *iref.key; + scoutfs_btree_put_iref(&iref); + last_seq = key.skts_trans_seq; + + } else if (ret == -ENOENT) { + last_seq = super->next_trans_seq; + ret = 0; + } + + le64_add_cpu(&last_seq, -1ULL); + trace_scoutfs_trans_seq_last(sb, rid, le64_to_cpu(last_seq)); + + up_read(&server->seq_rwsem); +out: + return scoutfs_net_response(sb, conn, cmd, id, ret, + &last_seq, sizeof(last_seq)); +} + +static int server_lock(struct super_block *sb, + struct scoutfs_net_connection *conn, + u8 cmd, u64 id, void *arg, u16 arg_len) +{ + u64 rid = scoutfs_net_client_rid(conn); + + if (arg_len != sizeof(struct scoutfs_net_lock)) + return -EINVAL; + + return scoutfs_lock_server_request(sb, rid, id, arg); +} + +static int lock_response(struct super_block *sb, + struct scoutfs_net_connection *conn, + void *resp, unsigned int resp_len, + int error, void *data) +{ + u64 rid = scoutfs_net_client_rid(conn); + + if (resp_len != sizeof(struct scoutfs_net_lock)) + return -EINVAL; + + return scoutfs_lock_server_response(sb, rid, resp); +} + +int scoutfs_server_lock_request(struct super_block *sb, u64 rid, + struct scoutfs_net_lock *nl) +{ + struct server_info *server = SCOUTFS_SB(sb)->server_info; + + return scoutfs_net_submit_request_node(sb, server->conn, rid, + SCOUTFS_NET_CMD_LOCK, + nl, sizeof(*nl), + lock_response, NULL, NULL); +} + +int scoutfs_server_lock_response(struct super_block *sb, u64 rid, u64 id, + struct scoutfs_net_lock_grant_response *gr) +{ + struct server_info *server = SCOUTFS_SB(sb)->server_info; + + return scoutfs_net_response_node(sb, server->conn, rid, + SCOUTFS_NET_CMD_LOCK, id, 0, + gr, sizeof(*gr)); +} + +static bool invalid_recover(struct scoutfs_net_lock_recover *nlr, + unsigned long bytes) +{ + return ((bytes < sizeof(*nlr)) || + (bytes != offsetof(struct scoutfs_net_lock_recover, + locks[le16_to_cpu(nlr->nr)]))); +} + +static int lock_recover_response(struct super_block *sb, + struct scoutfs_net_connection *conn, + void *resp, unsigned int resp_len, + int error, void *data) +{ + u64 rid = scoutfs_net_client_rid(conn); + + if (invalid_recover(resp, resp_len)) + return -EINVAL; + + return scoutfs_lock_server_recover_response(sb, rid, resp); +} + +int scoutfs_server_lock_recover_request(struct super_block *sb, u64 rid, + struct scoutfs_key *key) +{ + struct server_info *server = SCOUTFS_SB(sb)->server_info; + + return scoutfs_net_submit_request_node(sb, server->conn, rid, + SCOUTFS_NET_CMD_LOCK_RECOVER, + key, sizeof(*key), + lock_recover_response, + NULL, NULL); +} + +static int server_srch_get_compact(struct super_block *sb, + struct scoutfs_net_connection *conn, + u8 cmd, u64 id, void *arg, u16 arg_len) +{ + DECLARE_SERVER_INFO(sb, server); + u64 rid = scoutfs_net_client_rid(conn); + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->super; + struct scoutfs_srch_compact *sc = NULL; + int ret; + + if (arg_len != 0) { + ret = -EINVAL; + goto out; + } + + sc = kzalloc(sizeof(struct scoutfs_srch_compact), GFP_NOFS); + if (sc == NULL) { + ret = -ENOMEM; + goto out; + } + + ret = scoutfs_server_hold_commit(sb); + if (ret) + goto out; + + mutex_lock(&server->srch_mutex); + ret = scoutfs_srch_get_compact(sb, &server->alloc, &server->wri, + &super->srch_root, rid, sc); + mutex_unlock(&server->srch_mutex); + if (ret == 0 && sc->nr == 0) + ret = -ENOENT; + if (ret < 0) + goto apply; + + mutex_lock(&server->alloc_mutex); + ret = scoutfs_alloc_fill_list(sb, &server->alloc, &server->wri, + &sc->meta_avail, server->meta_avail, + SCOUTFS_SERVER_META_FILL_LO, + SCOUTFS_SERVER_META_FILL_TARGET) ?: + scoutfs_alloc_splice_list(sb, &server->alloc, &server->wri, + server->other_freed, &sc->meta_freed); + mutex_unlock(&server->alloc_mutex); + if (ret < 0) + goto apply; + + mutex_lock(&server->srch_mutex); + ret = scoutfs_srch_update_compact(sb, &server->alloc, &server->wri, + &super->srch_root, rid, sc); + mutex_unlock(&server->srch_mutex); + +apply: + ret = scoutfs_server_apply_commit(sb, ret); + WARN_ON_ONCE(ret < 0 && ret != -ENOENT); /* XXX leaked busy item */ +out: + ret = scoutfs_net_response(sb, conn, cmd, id, ret, + sc, sizeof(struct scoutfs_srch_compact)); + kfree(sc); + return ret; +} + +/* + * Commit the client's compaction. Their freed allocator contains the + * source srch files blocks that are currently in use which can't be + * available for allocation until after the commit. We move them into + * freed so they won't satisfy allocations. + */ +static int server_srch_commit_compact(struct super_block *sb, + struct scoutfs_net_connection *conn, + u8 cmd, u64 id, void *arg, u16 arg_len) +{ + DECLARE_SERVER_INFO(sb, server); + u64 rid = scoutfs_net_client_rid(conn); + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->super; + struct scoutfs_srch_compact *sc; + struct scoutfs_alloc_list_head av; + struct scoutfs_alloc_list_head fr; + int ret; + + if (arg_len != sizeof(struct scoutfs_srch_compact)) { + ret = -EINVAL; + goto out; + } + sc = arg; + + ret = scoutfs_server_hold_commit(sb); + if (ret) + goto out; + + mutex_lock(&server->srch_mutex); + ret = scoutfs_srch_commit_compact(sb, &server->alloc, &server->wri, + &super->srch_root, rid, sc, + &av, &fr); + mutex_unlock(&server->srch_mutex); + if (ret < 0) /* XXX very bad, leaks allocators */ + goto apply; + + /* reclaim allocators if they were set by _srch_commit_ */ + mutex_lock(&server->alloc_mutex); + ret = scoutfs_alloc_splice_list(sb, &server->alloc, &server->wri, + server->other_freed, &av) ?: + scoutfs_alloc_splice_list(sb, &server->alloc, &server->wri, + server->other_freed, &fr); + mutex_unlock(&server->alloc_mutex); +apply: + ret = scoutfs_server_apply_commit(sb, ret); +out: + WARN_ON(ret < 0); /* XXX leaks allocators */ + return scoutfs_net_response(sb, conn, cmd, id, ret, NULL, 0); +} + +static void init_mounted_client_key(struct scoutfs_key *key, u64 rid) +{ + *key = (struct scoutfs_key) { + .sk_zone = SCOUTFS_MOUNTED_CLIENT_ZONE, + .skmc_rid = cpu_to_le64(rid), + }; +} + +static int insert_mounted_client(struct super_block *sb, u64 rid, + u64 gr_flags) +{ + DECLARE_SERVER_INFO(sb, server); + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + struct scoutfs_mounted_client_btree_val mcv; + struct scoutfs_key key; + + init_mounted_client_key(&key, rid); + mcv.flags = 0; + if (gr_flags & SCOUTFS_NET_GREETING_FLAG_VOTER) + mcv.flags |= SCOUTFS_MOUNTED_CLIENT_VOTER; + + return scoutfs_btree_insert(sb, &server->alloc, &server->wri, + &super->mounted_clients, &key, &mcv, + sizeof(mcv)); +} + +/* + * Remove the record of a mounted client. The record can already be + * removed if we're processing a farewell on behalf of a client that + * already had a previous server process its farewell. + * + * When we remove the last mounted client that's voting we write a new + * quorum block with the updated unmount_barrier. + * + * The caller has to serialize with farewell processing. + */ +static int delete_mounted_client(struct super_block *sb, u64 rid) +{ + DECLARE_SERVER_INFO(sb, server); + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + struct scoutfs_key key; + int ret; + + init_mounted_client_key(&key, rid); + + ret = scoutfs_btree_delete(sb, &server->alloc, &server->wri, + &super->mounted_clients, &key); + if (ret == -ENOENT) + ret = 0; + + return ret; +} + +/* + * Remove all the busy items for srch compactions that the mount might + * have been responsible for and reclaim all their allocators. The freed + * allocator could still contain stable srch file blknos. + */ +static int cancel_srch_compact(struct super_block *sb, u64 rid) +{ + DECLARE_SERVER_INFO(sb, server); + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + struct scoutfs_alloc_list_head av; + struct scoutfs_alloc_list_head fr; + int ret; + + for (;;) { + mutex_lock(&server->srch_mutex); + ret = scoutfs_srch_cancel_compact(sb, &server->alloc, + &server->wri, + &super->srch_root, rid, + &av, &fr); + mutex_unlock(&server->srch_mutex); + if (ret < 0) { + if (ret == -ENOENT) + ret = 0; + break; + } + + mutex_lock(&server->alloc_mutex); + ret = scoutfs_alloc_splice_list(sb, &server->alloc, + &server->wri, + server->other_freed, &av) ?: + scoutfs_alloc_splice_list(sb, &server->alloc, + &server->wri, + server->other_freed, &fr); + mutex_unlock(&server->alloc_mutex); + if (WARN_ON_ONCE(ret < 0)) + break; + } + + return ret; +} + +/* + * Process an incoming greeting request in the server from the client. + * We try to send responses to failed greetings so that the sender can + * log some detail before shutting down. A failure to send a greeting + * response shuts down the connection. + * + * If a client reconnects they'll send their previously received + * serer_term in their greeting request. + * + * XXX The logic of this has gotten convoluted. The lock server can + * send a recovery request so it needs to be called after the core net + * greeting call enables messages. But we want the greeting reply to be + * sent first, so we currently queue it on the send queue before + * enabling messages. That means that a lot of errors that happen after + * the reply can't be sent to the client. They'll just see a disconnect + * and won't know what's happened. This all needs to be refactored. + */ +static int server_greeting(struct super_block *sb, + struct scoutfs_net_connection *conn, + u8 cmd, u64 id, void *arg, u16 arg_len) +{ + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + struct scoutfs_net_greeting *gr = arg; + struct scoutfs_net_greeting greet; + DECLARE_SERVER_INFO(sb, server); + __le64 umb = 0; + bool reconnecting; + bool first_contact; + bool farewell; + int ret = 0; + int err; + + if (arg_len != sizeof(struct scoutfs_net_greeting)) { + ret = -EINVAL; + goto send_err; + } + + if (gr->fsid != super->hdr.fsid) { + scoutfs_warn(sb, "client sent fsid 0x%llx, server has 0x%llx", + le64_to_cpu(gr->fsid), + le64_to_cpu(super->hdr.fsid)); + ret = -EINVAL; + goto send_err; + } + + if (gr->format_hash != super->format_hash) { + scoutfs_warn(sb, "client sent format 0x%llx, server has 0x%llx", + le64_to_cpu(gr->format_hash), + le64_to_cpu(super->format_hash)); + ret = -EINVAL; + goto send_err; + } + + if (gr->server_term == 0) { + ret = scoutfs_server_hold_commit(sb); + if (ret < 0) + goto send_err; + + spin_lock(&server->lock); + umb = super->unmount_barrier; + spin_unlock(&server->lock); + + mutex_lock(&server->farewell_mutex); + ret = insert_mounted_client(sb, le64_to_cpu(gr->rid), + le64_to_cpu(gr->flags)); + mutex_unlock(&server->farewell_mutex); + + ret = scoutfs_server_apply_commit(sb, ret); + queue_work(server->wq, &server->farewell_work); + } else { + umb = gr->unmount_barrier; + } + +send_err: + err = ret; + + greet.fsid = super->hdr.fsid; + greet.format_hash = super->format_hash; + greet.server_term = cpu_to_le64(server->term); + greet.unmount_barrier = umb; + greet.rid = gr->rid; + greet.flags = 0; + + /* queue greeting response to be sent first once messaging enabled */ + ret = scoutfs_net_response(sb, conn, cmd, id, err, + &greet, sizeof(greet)); + if (ret == 0 && err) + ret = err; + if (ret) + goto out; + + /* have the net core enable messaging and resend */ + reconnecting = gr->server_term != 0; + first_contact = le64_to_cpu(gr->server_term) != server->term; + if (gr->flags & cpu_to_le64(SCOUTFS_NET_GREETING_FLAG_FAREWELL)) + farewell = true; + else + farewell = false; + + scoutfs_net_server_greeting(sb, conn, le64_to_cpu(gr->rid), id, + reconnecting, first_contact, farewell); + + /* lock server might send recovery request */ + if (le64_to_cpu(gr->server_term) != server->term) { + + /* we're now doing two commits per greeting, not great */ + ret = scoutfs_server_hold_commit(sb); + if (ret) + goto out; + + ret = scoutfs_lock_server_greeting(sb, le64_to_cpu(gr->rid), + gr->server_term != 0); + ret = scoutfs_server_apply_commit(sb, ret); + if (ret) + goto out; + } + +out: + return ret; +} + +struct farewell_request { + struct list_head entry; + u64 net_id; + u64 rid; +}; + +static bool invalid_mounted_client_item(struct scoutfs_btree_item_ref *iref) +{ + return (iref->val_len != + sizeof(struct scoutfs_mounted_client_btree_val)); +} + +/* + * This work processes farewell requests asynchronously. Requests from + * voting clients can be held until only the final quorum remains and + * they've all sent farewell requests. + * + * When we remove the last mounted client record for the last voting + * client then we increase the unmount_barrier and write it to the super + * block. If voting clients don't get their farewell response they'll + * see the greater umount_barrier in the super and will know that their + * farewell has been processed and that they can exit. + * + * Responses that are waiting for clients who aren't voting are + * immediately sent. Clients that don't have a mounted client record + * have already had their farewell processed by another server and can + * proceed. + * + * Farewell responses are unique in that sending them causes the server + * to shutdown the connection to the client next time the socket + * disconnects. If the socket is destroyed before the client gets the + * response they'll reconnect and we'll see them as a brand new client + * who immediately sends a farewell. It'll be processed and it all + * works out. + * + * If this worker sees an error it assumes that this sever is done for + * and that another had better take its place. + */ +static void farewell_worker(struct work_struct *work) +{ + struct server_info *server = container_of(work, struct server_info, + farewell_work); + struct super_block *sb = server->sb; + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + struct scoutfs_mounted_client_btree_val *mcv; + struct farewell_request *tmp; + struct farewell_request *fw; + SCOUTFS_BTREE_ITEM_REF(iref); + unsigned int nr_unmounting = 0; + unsigned int nr_mounted = 0; + struct scoutfs_key key; + LIST_HEAD(reqs); + LIST_HEAD(send); + bool deleted = false; + bool voting; + bool more_reqs; + int ret; + + /* grab all the requests that are waiting */ + mutex_lock(&server->farewell_mutex); + list_splice_init(&server->farewell_requests, &reqs); + mutex_unlock(&server->farewell_mutex); + + /* count how many reqs requests are from voting clients */ + nr_unmounting = 0; + list_for_each_entry_safe(fw, tmp, &reqs, entry) { + init_mounted_client_key(&key, fw->rid); + ret = scoutfs_btree_lookup(sb, &super->mounted_clients, &key, + &iref); + if (ret == 0 && invalid_mounted_client_item(&iref)) { + scoutfs_btree_put_iref(&iref); + ret = -EIO; + } + if (ret < 0) { + if (ret == -ENOENT) { + list_move_tail(&fw->entry, &send); + continue; + } + goto out; + } + + mcv = iref.val; + voting = (mcv->flags & SCOUTFS_MOUNTED_CLIENT_VOTER) != 0; + scoutfs_btree_put_iref(&iref); + + if (!voting) { + list_move_tail(&fw->entry, &send); + continue; + } + + nr_unmounting++; + } + + /* see how many mounted clients could vote for quorum */ + init_mounted_client_key(&key, 0); + for (;;) { + ret = scoutfs_btree_next(sb, &super->mounted_clients, &key, + &iref); + if (ret == 0 && invalid_mounted_client_item(&iref)) { + scoutfs_btree_put_iref(&iref); + ret = -EIO; + } + if (ret != 0) { + if (ret == -ENOENT) + break; + goto out; + } + + key = *iref.key; + mcv = iref.val; + + if (mcv->flags & SCOUTFS_MOUNTED_CLIENT_VOTER) + nr_mounted++; + + scoutfs_btree_put_iref(&iref); + scoutfs_key_inc(&key); + } + + /* send as many responses as we can to maintain quorum */ + while ((fw = list_first_entry_or_null(&reqs, struct farewell_request, + entry)) && + (nr_mounted > super->quorum_count || + nr_unmounting >= nr_mounted)) { + + list_move_tail(&fw->entry, &send); + nr_mounted--; + nr_unmounting--; + deleted = true; + } + + /* process and send farewell responses */ + list_for_each_entry_safe(fw, tmp, &send, entry) { + ret = scoutfs_server_hold_commit(sb); + if (ret) + goto out; + + ret = scoutfs_lock_server_farewell(sb, fw->rid) ?: + remove_trans_seq(sb, fw->rid) ?: + reclaim_log_trees(sb, fw->rid) ?: + delete_mounted_client(sb, fw->rid) ?: + cancel_srch_compact(sb, fw->rid); + + ret = scoutfs_server_apply_commit(sb, ret); + if (ret) + goto out; + } + + /* update the unmount barrier if we deleted all voting clients */ + if (deleted && nr_mounted == 0) { + ret = scoutfs_server_hold_commit(sb); + if (ret) + goto out; + + le64_add_cpu(&super->unmount_barrier, 1); + + ret = scoutfs_server_apply_commit(sb, ret); + if (ret) + goto out; + } + + /* and finally send all the responses */ + list_for_each_entry_safe(fw, tmp, &send, entry) { + + ret = scoutfs_net_response_node(sb, server->conn, fw->rid, + SCOUTFS_NET_CMD_FAREWELL, + fw->net_id, 0, NULL, 0); + if (ret) + break; + + list_del_init(&fw->entry); + kfree(fw); + } + + ret = 0; +out: + mutex_lock(&server->farewell_mutex); + more_reqs = !list_empty(&server->farewell_requests); + list_splice_init(&reqs, &server->farewell_requests); + list_splice_init(&send, &server->farewell_requests); + mutex_unlock(&server->farewell_mutex); + + if (ret < 0) + stop_server(server); + else if (more_reqs && !server->shutting_down) + queue_work(server->wq, &server->farewell_work); +} + +static void free_farewell_requests(struct super_block *sb, u64 rid) +{ + struct server_info *server = SCOUTFS_SB(sb)->server_info; + struct farewell_request *tmp; + struct farewell_request *fw; + + mutex_lock(&server->farewell_mutex); + list_for_each_entry_safe(fw, tmp, &server->farewell_requests, entry) { + if (rid == 0 || fw->rid == rid) { + list_del_init(&fw->entry); + kfree(fw); + } + } + mutex_unlock(&server->farewell_mutex); +} + +/* + * The server is receiving a farewell message from a client that is + * unmounting. It won't send any more requests and once it receives our + * response it will not reconnect. + * + * XXX we should make sure that all our requests to the client have finished + * before we respond. Locking will have its own messaging for orderly + * shutdown. That leaves compaction which will be addressed as part of + * the larger work of recovering compactions that were in flight when + * a client crashed. + */ +static int server_farewell(struct super_block *sb, + struct scoutfs_net_connection *conn, + u8 cmd, u64 id, void *arg, u16 arg_len) +{ + struct server_info *server = SCOUTFS_SB(sb)->server_info; + u64 rid = scoutfs_net_client_rid(conn); + struct farewell_request *fw; + + if (arg_len != 0) + return -EINVAL; + + /* XXX tear down if we fence, or if we shut down */ + + fw = kmalloc(sizeof(struct farewell_request), GFP_NOFS); + if (fw == NULL) + return -ENOMEM; + + fw->rid = rid; + fw->net_id = id; + + mutex_lock(&server->farewell_mutex); + list_add_tail(&fw->entry, &server->farewell_requests); + mutex_unlock(&server->farewell_mutex); + + queue_work(server->wq, &server->farewell_work); + + /* response will be sent later */ + return 0; +} + +static scoutfs_net_request_t server_req_funcs[] = { + [SCOUTFS_NET_CMD_GREETING] = server_greeting, + [SCOUTFS_NET_CMD_ALLOC_INODES] = server_alloc_inodes, + [SCOUTFS_NET_CMD_GET_LOG_TREES] = server_get_log_trees, + [SCOUTFS_NET_CMD_COMMIT_LOG_TREES] = server_commit_log_trees, + [SCOUTFS_NET_CMD_GET_ROOTS] = server_get_roots, + [SCOUTFS_NET_CMD_ADVANCE_SEQ] = server_advance_seq, + [SCOUTFS_NET_CMD_GET_LAST_SEQ] = server_get_last_seq, + [SCOUTFS_NET_CMD_LOCK] = server_lock, + [SCOUTFS_NET_CMD_SRCH_GET_COMPACT] = server_srch_get_compact, + [SCOUTFS_NET_CMD_SRCH_COMMIT_COMPACT] = server_srch_commit_compact, + [SCOUTFS_NET_CMD_FAREWELL] = server_farewell, +}; + +static void server_notify_up(struct super_block *sb, + struct scoutfs_net_connection *conn, + void *info, u64 rid) +{ + struct server_client_info *sci = info; + DECLARE_SERVER_INFO(sb, server); + + if (rid != 0) { + sci->rid = rid; + spin_lock(&server->lock); + list_add_tail(&sci->head, &server->clients); + server->nr_clients++; + trace_scoutfs_server_client_up(sb, rid, server->nr_clients); + spin_unlock(&server->lock); + } +} + +static void server_notify_down(struct super_block *sb, + struct scoutfs_net_connection *conn, + void *info, u64 rid) +{ + struct server_client_info *sci = info; + DECLARE_SERVER_INFO(sb, server); + + if (rid != 0) { + spin_lock(&server->lock); + list_del_init(&sci->head); + server->nr_clients--; + trace_scoutfs_server_client_down(sb, rid, + server->nr_clients); + spin_unlock(&server->lock); + + free_farewell_requests(sb, rid); + } else { + stop_server(server); + } +} + +static void scoutfs_server_worker(struct work_struct *work) +{ + struct server_info *server = container_of(work, struct server_info, + work); + struct super_block *sb = server->sb; + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->super; + struct scoutfs_net_connection *conn = NULL; + DECLARE_WAIT_QUEUE_HEAD(waitq); + struct sockaddr_in sin; + LIST_HEAD(conn_list); + u64 max_vers; + int ret; + int err; + + trace_scoutfs_server_work_enter(sb, 0, 0); + + sin = server->listen_sin; + + scoutfs_info(sb, "server setting up at "SIN_FMT, SIN_ARG(&sin)); + + conn = scoutfs_net_alloc_conn(sb, server_notify_up, server_notify_down, + sizeof(struct server_client_info), + server_req_funcs, "server"); + if (!conn) { + ret = -ENOMEM; + goto out; + } + + ret = scoutfs_net_bind(sb, conn, &sin); + if (ret) { + scoutfs_err(sb, "server failed to bind to "SIN_FMT", err %d%s", + SIN_ARG(&sin), ret, + ret == -EADDRNOTAVAIL ? " (Bad address?)" + : ""); + goto out; + } + + if (ret) + goto out; + + /* start up the server subsystems before accepting */ + ret = scoutfs_read_super(sb, super); + if (ret < 0) + goto shutdown; + + set_roots(server, &super->fs_root, &super->logs_root, + &super->srch_root); + scoutfs_block_writer_init(sb, &server->wri); + + /* prepare server alloc for this transaction, larger first */ + if (le64_to_cpu(super->server_meta_avail[0].total_nr) < + le64_to_cpu(super->server_meta_avail[1].total_nr)) + server->other_ind = 0; + else + server->other_ind = 1; + scoutfs_alloc_init(&server->alloc, + &super->server_meta_avail[server->other_ind ^ 1], + &super->server_meta_freed[server->other_ind ^ 1]); + server->other_avail = &super->server_meta_avail[server->other_ind]; + server->other_freed = &super->server_meta_freed[server->other_ind]; + + /* use largest meta_alloc to start */ + server->meta_avail = &super->meta_alloc[0]; + server->meta_freed = &super->meta_alloc[1]; + if (le64_to_cpu(server->meta_freed->total_len) > + le64_to_cpu(server->meta_avail->total_len)) + swap(server->meta_avail, server->meta_freed); + + ret = scoutfs_forest_get_max_vers(sb, super, &max_vers); + if (ret) { + scoutfs_err(sb, "server couldn't find max item vers: %d", ret); + goto shutdown; + } + + ret = scoutfs_lock_server_setup(sb, &server->alloc, &server->wri, + max_vers); + if (ret) + goto shutdown; + + /* + * Write our address in the super before it's possible for net + * processing to start writing the super as part of + * transactions. In theory clients could be trying to connect + * to our address without having seen it in the super (maybe + * they saw it a long time ago). + */ + scoutfs_addr_from_sin(&super->server_addr, &sin); + super->quorum_server_term = cpu_to_le64(server->term); + ret = scoutfs_write_super(sb, super); + if (ret < 0) + goto shutdown; + + /* start accepting connections and processing work */ + server->conn = conn; + scoutfs_net_listen(sb, conn); + + scoutfs_info(sb, "server ready at "SIN_FMT, SIN_ARG(&sin)); + complete(&server->start_comp); + + /* wait_event/wake_up provide barriers */ + wait_event_interruptible(server->waitq, server->shutting_down); + +shutdown: + scoutfs_info(sb, "server shutting down at "SIN_FMT, SIN_ARG(&sin)); + /* wait for request processing */ + scoutfs_net_shutdown(sb, conn); + /* wait for commit queued by request processing */ + flush_work(&server->commit_work); + server->conn = NULL; + + scoutfs_lock_server_destroy(sb); + +out: + scoutfs_quorum_clear_leader(sb); + scoutfs_net_free_conn(sb, conn); + + scoutfs_info(sb, "server stopped at "SIN_FMT, SIN_ARG(&sin)); + trace_scoutfs_server_work_exit(sb, 0, ret); + + /* + * Always try to clear our presence in the super so that we're + * not fenced. We do this last because other mounts will try to + * reach quorum the moment they see zero here. The later we do + * this the longer we have to finish shutdown while clients + * timeout. + */ + err = scoutfs_read_super(sb, super); + if (err == 0) { + super->quorum_fenced_term = cpu_to_le64(server->term); + memset(&super->server_addr, 0, sizeof(super->server_addr)); + err = scoutfs_write_super(sb, super); + } + if (err < 0) { + scoutfs_err(sb, "failed to clear election term %llu at "SIN_FMT", this mount could be fenced", + server->term, SIN_ARG(&sin)); + } + + server->err = ret; + complete(&server->start_comp); +} + +/* + * Wait for the server to successfully start. If this returns error then + * the super block's fence_term has been set to the new server's term so + * that it won't be fenced. + */ +int scoutfs_server_start(struct super_block *sb, struct sockaddr_in *sin, + u64 term) +{ + DECLARE_SERVER_INFO(sb, server); + + server->err = 0; + server->shutting_down = false; + server->listen_sin = *sin; + server->term = term; + init_completion(&server->start_comp); + + queue_work(server->wq, &server->work); + + wait_for_completion(&server->start_comp); + return server->err; +} + +/* + * Start shutdown on the server but don't want for it to finish. + */ +void scoutfs_server_abort(struct super_block *sb) +{ + DECLARE_SERVER_INFO(sb, server); + + stop_server(server); +} + +/* + * Once the server is stopped we give the caller our election info + * which might have been modified while we were running. + */ +void scoutfs_server_stop(struct super_block *sb) +{ + DECLARE_SERVER_INFO(sb, server); + + stop_server(server); + /* XXX not sure both are needed */ + cancel_work_sync(&server->work); + cancel_work_sync(&server->commit_work); +} + +int scoutfs_server_setup(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct server_info *server; + + server = kzalloc(sizeof(struct server_info), GFP_KERNEL); + if (!server) + return -ENOMEM; + + server->sb = sb; + spin_lock_init(&server->lock); + init_waitqueue_head(&server->waitq); + INIT_WORK(&server->work, scoutfs_server_worker); + init_rwsem(&server->commit_rwsem); + init_llist_head(&server->commit_waiters); + INIT_WORK(&server->commit_work, scoutfs_server_commit_func); + init_rwsem(&server->seq_rwsem); + INIT_LIST_HEAD(&server->clients); + mutex_init(&server->farewell_mutex); + INIT_LIST_HEAD(&server->farewell_requests); + INIT_WORK(&server->farewell_work, farewell_worker); + mutex_init(&server->alloc_mutex); + mutex_init(&server->logs_mutex); + mutex_init(&server->srch_mutex); + seqcount_init(&server->roots_seqcount); + + server->wq = alloc_workqueue("scoutfs_server", + WQ_UNBOUND | WQ_NON_REENTRANT, 0); + if (!server->wq) { + kfree(server); + return -ENOMEM; + } + + sbi->server_info = server; + return 0; +} + +/* + * The caller should have already stopped but we do the same just in + * case. + */ +void scoutfs_server_destroy(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct server_info *server = sbi->server_info; + + if (server) { + stop_server(server); + + /* wait for server work to wait for everything to shut down */ + cancel_work_sync(&server->work); + /* recv work/compaction could have left commit_work queued */ + cancel_work_sync(&server->commit_work); + + /* pending farewell requests are another server's problem */ + cancel_work_sync(&server->farewell_work); + free_farewell_requests(sb, 0); + + trace_scoutfs_server_workqueue_destroy(sb, 0, 0); + destroy_workqueue(server->wq); + + kfree(server); + sbi->server_info = NULL; + } +} diff --git a/kmod/src/server.h b/kmod/src/server.h new file mode 100644 index 00000000..274a66ea --- /dev/null +++ b/kmod/src/server.h @@ -0,0 +1,80 @@ +#ifndef _SCOUTFS_SERVER_H_ +#define _SCOUTFS_SERVER_H_ + +#define SI4_FMT "%u.%u.%u.%u:%u" + +#define si4_trace_define(name) \ + __field(__u32, name##_addr) \ + __field(__u16, name##_port) + +#define si4_trace_assign(name, sin) \ +do { \ + __typeof__(sin) _sin = (sin); \ + \ + __entry->name##_addr = be32_to_cpu(_sin->sin_addr.s_addr); \ + __entry->name##_port = be16_to_cpu(_sin->sin_port); \ +} while(0) + +#define si4_trace_args(name) \ + (__entry->name##_addr >> 24), \ + (__entry->name##_addr >> 16) & 255, \ + (__entry->name##_addr >> 8) & 255, \ + __entry->name##_addr & 255, \ + __entry->name##_port + +#define SNH_FMT \ + "seq %llu recv_seq %llu id %llu data_len %u cmd %u flags 0x%x error %u" +#define SNH_ARG(nh) \ + le64_to_cpu((nh)->seq), le64_to_cpu((nh)->recv_seq), \ + le64_to_cpu((nh)->id), le16_to_cpu((nh)->data_len), (nh)->cmd, \ + (nh)->flags, (nh)->error + +#define snh_trace_define(name) \ + __field(__u64, name##_seq) \ + __field(__u64, name##_recv_seq) \ + __field(__u64, name##_id) \ + __field(__u16, name##_data_len) \ + __field(__u8, name##_cmd) \ + __field(__u8, name##_flags) \ + __field(__u8, name##_error) + +#define snh_trace_assign(name, nh) \ +do { \ + __typeof__(nh) _nh = (nh); \ + \ + __entry->name##_seq = le64_to_cpu(_nh->seq); \ + __entry->name##_recv_seq = le64_to_cpu(_nh->recv_seq); \ + __entry->name##_id = le64_to_cpu(_nh->id); \ + __entry->name##_data_len = le16_to_cpu(_nh->data_len); \ + __entry->name##_cmd = _nh->cmd; \ + __entry->name##_flags = _nh->flags; \ + __entry->name##_error = _nh->error; \ +} while (0) + +#define snh_trace_args(name) \ + __entry->name##_seq, __entry->name##_recv_seq, __entry->name##_id, \ + __entry->name##_data_len, __entry->name##_cmd, __entry->name##_flags, \ + __entry->name##_error + +int scoutfs_server_lock_request(struct super_block *sb, u64 rid, + struct scoutfs_net_lock *nl); +int scoutfs_server_lock_response(struct super_block *sb, u64 rid, u64 id, + struct scoutfs_net_lock_grant_response *gr); +int scoutfs_server_lock_recover_request(struct super_block *sb, u64 rid, + struct scoutfs_key *key); +void scoutfs_server_get_roots(struct super_block *sb, + struct scoutfs_net_roots *roots); +int scoutfs_server_hold_commit(struct super_block *sb); +int scoutfs_server_apply_commit(struct super_block *sb, int err); + +struct sockaddr_in; +struct scoutfs_quorum_elected_info; +int scoutfs_server_start(struct super_block *sb, struct sockaddr_in *sin, + u64 term); +void scoutfs_server_abort(struct super_block *sb); +void scoutfs_server_stop(struct super_block *sb); + +int scoutfs_server_setup(struct super_block *sb); +void scoutfs_server_destroy(struct super_block *sb); + +#endif diff --git a/kmod/src/sort_priv.c b/kmod/src/sort_priv.c new file mode 100644 index 00000000..2acc0802 --- /dev/null +++ b/kmod/src/sort_priv.c @@ -0,0 +1,71 @@ +/* + * A copy of sort() from upstream with a priv argument that's passed + * to comparison, like list_sort(). + */ + +/* ------------------------ */ + +/* + * A fast, small, non-recursive O(nlog n) sort for the Linux kernel + * + * Jan 23 2005 Matt Mackall + */ + +#include +#include +#include +#include +#include "sort_priv.h" + +/** + * sort - sort an array of elements + * @priv: caller's pointer to pass to comparison and swap functions + * @base: pointer to data to sort + * @num: number of elements + * @size: size of each element + * @cmp_func: pointer to comparison function + * @swap_func: pointer to swap function or NULL + * + * This function does a heapsort on the given array. You may provide a + * swap_func function optimized to your element type. + * + * Sorting time is O(n log n) both on average and worst-case. While + * qsort is about 20% faster on average, it suffers from exploitable + * O(n*n) worst-case behavior and extra memory requirements that make + * it less suitable for kernel use. + */ + +void sort_priv(void *priv, void *base, size_t num, size_t size, + int (*cmp_func)(void *priv, const void *, const void *), + void (*swap_func)(void *priv, void *, void *, int size)) +{ + /* pre-scale counters for performance */ + int i = (num/2 - 1) * size, n = num * size, c, r; + + /* heapify */ + for ( ; i >= 0; i -= size) { + for (r = i; r * 2 + size < n; r = c) { + c = r * 2 + size; + if (c < n - size && + cmp_func(priv, base + c, base + c + size) < 0) + c += size; + if (cmp_func(priv, base + r, base + c) >= 0) + break; + swap_func(priv, base + r, base + c, size); + } + } + + /* sort */ + for (i = n - size; i > 0; i -= size) { + swap_func(priv, base, base + i, size); + for (r = 0; r * 2 + size < i; r = c) { + c = r * 2 + size; + if (c < i - size && + cmp_func(priv, base + c, base + c + size) < 0) + c += size; + if (cmp_func(priv, base + r, base + c) >= 0) + break; + swap_func(priv, base + r, base + c, size); + } + } +} diff --git a/kmod/src/sort_priv.h b/kmod/src/sort_priv.h new file mode 100644 index 00000000..c5fde547 --- /dev/null +++ b/kmod/src/sort_priv.h @@ -0,0 +1,8 @@ +#ifndef _SCOUTFS_SORT_PRIV_H_ +#define _SCOUTFS_SORT_PRIV_H_ + +void sort_priv(void *priv, void *base, size_t num, size_t size, + int (*cmp_func)(void *priv, const void *, const void *), + void (*swap_func)(void *priv, void *, void *, int size)); + +#endif diff --git a/kmod/src/spbm.c b/kmod/src/spbm.c new file mode 100644 index 00000000..d2ff89eb --- /dev/null +++ b/kmod/src/spbm.c @@ -0,0 +1,158 @@ +/* + * Copyright (C) 2018 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include +#include +#include + +#include "spbm.h" + +#define SPBM_BITS 128 +#define SPBM_SHIFT ilog2(SPBM_BITS) +#define SPBM_MASK ((u64)SPBM_BITS - 1) +#define SPBM_LONGS (SPBM_BITS / BITS_PER_LONG) + +/* + * Maintain a sparse bitmap in an rbtree. Setting bits can allocate and + * fail but clearing will always succeed. Locking is left up to the + * caller. + */ + +struct spbm_node { + struct rb_node node; + u64 index; + unsigned long bits[SPBM_LONGS]; +}; + +void scoutfs_spbm_init(struct scoutfs_spbm *spbm) +{ + BUILD_BUG_ON(!is_power_of_2(SPBM_BITS)); + + spbm->root = RB_ROOT; +} + +bool scoutfs_spbm_empty(struct scoutfs_spbm *spbm) +{ + return RB_EMPTY_ROOT(&spbm->root); +} + +enum spbm_flags { + /* if a node isn't found then return an allocated new node */ + SPBM_FIND_ALLOC = (1 << 0), +}; +static struct spbm_node *find_node(struct scoutfs_spbm *spbm, u64 index, + int flags) +{ + struct rb_node *parent; + struct rb_node **node; + struct spbm_node *sn; + + node = &spbm->root.rb_node; + parent = NULL; + sn = NULL; + while (*node) { + parent = *node; + sn = container_of(*node, struct spbm_node, node); + + if (index < sn->index) { + node = &(*node)->rb_left; + } else if (index > sn->index) { + node = &(*node)->rb_right; + } else { + break; + } + + sn = NULL; + } + + if (!sn && (flags & SPBM_FIND_ALLOC)) { + sn = kzalloc(sizeof(struct spbm_node), GFP_NOFS); + if (sn) { + sn->index = index; + rb_link_node(&sn->node, parent, node); + rb_insert_color(&sn->node, &spbm->root); + } + } + + return sn; +} + +static void calc_index_nr(u64 *index, int *nr, u64 bit) +{ + *index = bit >> SPBM_SHIFT; + *nr = bit & SPBM_MASK; +} + +int scoutfs_spbm_set(struct scoutfs_spbm *spbm, u64 bit) +{ + struct spbm_node *sn; + u64 index; + int nr; + + calc_index_nr(&index, &nr, bit); + + sn = find_node(spbm, index, SPBM_FIND_ALLOC); + if (!sn) + return -ENOMEM; + + set_bit(nr, sn->bits); + + return 0; +} + +int scoutfs_spbm_test(struct scoutfs_spbm *spbm, u64 bit) +{ + struct spbm_node *sn; + u64 index; + int nr; + + calc_index_nr(&index, &nr, bit); + + sn = find_node(spbm, index, 0); + if (sn) + return !!test_bit(nr, sn->bits); + + return 0; +} + +static void free_node(struct scoutfs_spbm *spbm, struct spbm_node *sn) +{ + rb_erase(&sn->node, &spbm->root); + kfree(sn); +} + +void scoutfs_spbm_clear(struct scoutfs_spbm *spbm, u64 bit) +{ + struct spbm_node *sn; + u64 index; + int nr; + + calc_index_nr(&index, &nr, bit); + + sn = find_node(spbm, index, 0); + if (sn) { + clear_bit(nr, sn->bits); + if (bitmap_empty(sn->bits, SPBM_BITS)) + free_node(spbm, sn); + } +} + +void scoutfs_spbm_destroy(struct scoutfs_spbm *spbm) +{ + struct spbm_node *sn; + struct spbm_node *pos; + + rbtree_postorder_for_each_entry_safe(sn, pos, &spbm->root, node) + free_node(spbm, sn); +} diff --git a/kmod/src/spbm.h b/kmod/src/spbm.h new file mode 100644 index 00000000..7d9dca21 --- /dev/null +++ b/kmod/src/spbm.h @@ -0,0 +1,16 @@ +#ifndef _SCOUTFS_SPBM_H_ +#define _SCOUTFS_SPBM_H_ + +struct scoutfs_spbm { + struct rb_root root; +}; + +void scoutfs_spbm_init(struct scoutfs_spbm *spbm); +bool scoutfs_spbm_empty(struct scoutfs_spbm *spbm); +void scoutfs_spbm_destroy(struct scoutfs_spbm *spbm); + +int scoutfs_spbm_set(struct scoutfs_spbm *spbm, u64 bit); +int scoutfs_spbm_test(struct scoutfs_spbm *spbm, u64 bit); +void scoutfs_spbm_clear(struct scoutfs_spbm *spbm, u64 bit); + +#endif diff --git a/kmod/src/srch.c b/kmod/src/srch.c new file mode 100644 index 00000000..4c361dd1 --- /dev/null +++ b/kmod/src/srch.c @@ -0,0 +1,2331 @@ +/* + * Copyright (C) 2020 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "super.h" +#include "format.h" +#include "counters.h" +#include "block.h" +#include "alloc.h" +#include "srch.h" +#include "btree.h" +#include "spbm.h" +#include "client.h" +#include "scoutfs_trace.h" + +/* + * This srch subsystem gives us a way to find inodes that have a given + * tagged xattr set. It's designed for an xattr population that is + * orders of magnitudes larger than the file population, is updated much + * more frequently than it is searched, and can have slightly relaxed + * consistency requirements so that searches don't have to serialize + * with updates through locking. + * + * A srch entry is logged every time a .srch. xattr is created or + * deleted. Commits append entries to a growing srch log file along + * with the item btree and allocator block structures they're modifying. + * + * The server regularly rotates these growing log files so that they + * don't exceed a given size. Once there are enough log files they're + * all read and their sorted entries are written to a larger sorted + * file. Once there are enough sorted files they're all read and their + * combined sorted entries are written to a larger file, and so on. + * + * Searches combine all the entries read from unsorted log files and + * binary searches of larger sorted files to come up with the candidate + * inodes that probably contain the given named .srch. xattr. + * + * Searches read rotated log files and sorted files which have been + * committed. There is nothing protecting their blocks from being + * re-allocated and re-written. Search can restart by checking the + * btree for the current set of files. Compaction reads log files which + * are protected from other compactions by the persistent busy items + * created by the server. Compaction won't see it's blocks reused out + * from under it, but it can encounter stale cached blocks that need to + * be invalidated. + */ + +struct srch_info { + struct super_block *sb; + atomic_t shutdown; + struct workqueue_struct *workq; + struct delayed_work compact_dwork; +}; + +#define DECLARE_SRCH_INFO(sb, name) \ + struct srch_info *name = SCOUTFS_SB(sb)->srch_info + +#define SRE_FMT "%016llx.%llu.%llu" +#define SRE_ARG(sre) \ + le64_to_cpu((sre)->hash), le64_to_cpu((sre)->ino), \ + le64_to_cpu((sre)->id) + +/* + * Compactions dirty radix allocator blocks, file radix parent blocks, + * and especially srch file blocks. The files can get enormous and we + * can't have compactions OOM the box but they're meant to be large + * streaming operations, so we only stop and write out dirty blocks in + * large chunks. + */ +#define SRCH_COMPACT_DIRTY_LIMIT_BYTES (32 * 1024 * 1024) + +static int sre_cmp(const struct scoutfs_srch_entry *a, + const struct scoutfs_srch_entry *b) +{ + return scoutfs_cmp_u64s(le64_to_cpu(a->hash), le64_to_cpu(b->hash)) ?: + scoutfs_cmp_u64s(le64_to_cpu(a->ino), le64_to_cpu(b->ino)) ?: + scoutfs_cmp_u64s(le64_to_cpu(a->id), le64_to_cpu(b->id)); +} + +static void sre_inc(struct scoutfs_srch_entry *sre) +{ + le64_add_cpu(&sre->id, 1); + if (sre->id != 0) + return; + le64_add_cpu(&sre->ino, 1); + if (sre->ino != 0) + return; + le64_add_cpu(&sre->hash, 1); +} + +static void sre_dec(struct scoutfs_srch_entry *sre) +{ + le64_add_cpu(&sre->id, -1); + if (sre->id != cpu_to_le64(U64_MAX)) + return; + le64_add_cpu(&sre->ino, -1); + if (sre->ino != cpu_to_le64(U64_MAX)) + return; + le64_add_cpu(&sre->hash, -1); +} + +/* + * srch items are first grouped by type and we have log files, sorted + * files, and busy compactions. + */ +static void init_srch_key(struct scoutfs_key *key, int type, + u64 major, u64 minor) +{ + *key = (struct scoutfs_key) { + .sk_zone = SCOUTFS_SRCH_ZONE, + .sk_type = type, + ._sk_second = cpu_to_le64(major), + ._sk_third = cpu_to_le64(minor), + }; +} + +/* + * The caller has ensured that there is space for a full word at the + * buf. Only the set low order bytes will be used. The clear high + * order bytes will be overwritten in the future and ignored in the + * final encoding in the block. + */ +static int encode_u64(__le64 *buf, u64 val) +{ + int bytes; + + val = (val << 1) ^ ((s64)val >> 63); /* shift sign extend */ + bytes = (fls64(val) + 7) >> 3; + + put_unaligned_le64(val, buf); + return bytes; +} + +/* shifting by width is undefined :/ */ +#define BYTE_MASK(b) ((1ULL << (b << 3)) - 1) +static u64 byte_masks[] = { + 0, BYTE_MASK(1), BYTE_MASK(2), BYTE_MASK(3), + BYTE_MASK(4), BYTE_MASK(5), BYTE_MASK(6), BYTE_MASK(7), U64_MAX, +}; + +static u64 decode_u64(void *buf, int bytes) +{ + u64 val = get_unaligned_le64(buf) & byte_masks[bytes]; + + return (val >> 1) ^ (-(val & 1)); +} + +/* + * Encode an entry at the offset in the block. Leave room for the + * lengths short, encode the diff of the encoded entry from the + * previous, then update the length short with the length of each + * encoded diff. The caller ensures that there's room for a full size + * entry at position in the block. + */ +static int encode_entry(void *buf, struct scoutfs_srch_entry *sre, + struct scoutfs_srch_entry *prev) +{ + u64 diffs[] = { + le64_to_cpu(sre->hash) - le64_to_cpu(prev->hash), + le64_to_cpu(sre->ino) - le64_to_cpu(prev->ino), + le64_to_cpu(sre->id) - le64_to_cpu(prev->id), + }; + u16 lengths = 0; + int bytes; + int tot = 2; + int i; + + for (i = 0; i < ARRAY_SIZE(diffs); i++) { + bytes = encode_u64(buf + tot, diffs[i]); + lengths |= bytes << (i << 2); + tot += bytes; + } + + put_unaligned_le16(lengths, buf); + + return tot; +} + +/* + * Decode an entry from the offset of the block. Load the length short + * and decode the bytes of diffs and apply them to the previous entry. + * The caller ensures that we won't read off the end of block if we were + * to try and decode a full size set of diffs. + */ +static int decode_entry(void *buf, struct scoutfs_srch_entry *sre, + struct scoutfs_srch_entry *prev) +{ + u64 diffs[3]; + u16 lengths; + int bytes; + int tot; + int i; + + lengths = get_unaligned_le16(buf); + tot = 2; + + for (i = 0; i < ARRAY_SIZE(diffs); i++) { + bytes = min_t(int, 8, lengths & 15); + diffs[i] = decode_u64(buf + tot, bytes); + tot += bytes; + lengths >>= 4; + } + + sre->hash = cpu_to_le64(le64_to_cpu(prev->hash) + diffs[0]); + sre->ino = cpu_to_le64(le64_to_cpu(prev->ino) + diffs[1]); + sre->id = cpu_to_le64(le64_to_cpu(prev->id) + diffs[2]); + + return tot; +} + +/* return refs ind to traverse through parent at level to blk */ +static int calc_ref_ind(u64 blk, int level) +{ + int ind; + int i; + + BUG_ON(level < 1); + + for (i = 1; i <= level; i++) + blk = div_u64_rem(blk, SCOUTFS_SRCH_PARENT_REFS, &ind); + + return ind; +} + +static u8 height_for_blk(u64 blk) +{ + u64 total = SCOUTFS_SRCH_PARENT_REFS; + int hei = 2; + + if (blk == 0) + return 1; + + while (blk >= total) { + hei++; + total *= SCOUTFS_SRCH_PARENT_REFS; + } + + return hei; +} + +static void init_file_block(struct super_block *sb, struct scoutfs_block *bl, + int level) +{ + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + struct scoutfs_block_header *hdr; + + /* don't leak uninit kernel mem.. block should do this for us? */ + memset(bl->data, 0, SCOUTFS_BLOCK_LG_SIZE); + + hdr = bl->data; + hdr->fsid = super->hdr.fsid; + hdr->blkno = cpu_to_le64(bl->blkno); + prandom_bytes(&hdr->seq, sizeof(hdr->seq)); + + if (level) + hdr->magic = cpu_to_le32(SCOUTFS_BLOCK_MAGIC_SRCH_PARENT); + else + hdr->magic = cpu_to_le32(SCOUTFS_BLOCK_MAGIC_SRCH_BLOCK); +} + +/* + * This is operating on behalf of writers writing into private files and + * readers who could see stale blocks. We can find stale cached blocks + * and should retry the read ourselves after invalidating, but if we hit + * stale blocks on disk then we have to return to the caller who can + * decide to return errors or retry. + */ +static int read_srch_block(struct super_block *sb, + struct scoutfs_block_writer *wri, int level, + struct scoutfs_srch_ref *ref, + struct scoutfs_block **bl_ret) +{ + struct scoutfs_block *bl; + int retries = 0; + int ret = 0; + int mag; + + mag = level ? SCOUTFS_BLOCK_MAGIC_SRCH_PARENT : + SCOUTFS_BLOCK_MAGIC_SRCH_BLOCK; +retry: + bl = scoutfs_block_read(sb, le64_to_cpu(ref->blkno)); + if (!IS_ERR_OR_NULL(bl) && + !scoutfs_block_consistent_ref(sb, bl, ref->seq, ref->blkno, mag)) { + + scoutfs_inc_counter(sb, srch_inconsistent_ref); + scoutfs_block_writer_forget(sb, wri, bl); + scoutfs_block_invalidate(sb, bl); + scoutfs_block_put(sb, bl); + bl = NULL; + + if (retries++ == 0) + goto retry; + + bl = ERR_PTR(-ESTALE); + scoutfs_inc_counter(sb, srch_read_stale); + } + if (IS_ERR(bl)) { + ret = PTR_ERR(bl); + bl = NULL; + } + + *bl_ret = bl; + return ret; +} + +/* + * Give the caller a read-only reference to the block along the path to + * the logical block at the given level. This shouldn't be called on an + * empty root. + */ +static int read_path_block(struct super_block *sb, + struct scoutfs_block_writer *wri, + struct scoutfs_srch_file *sfl, + u64 blk, int at_level, + struct scoutfs_block **bl_ret) +{ + struct scoutfs_block *bl = NULL; + struct scoutfs_srch_parent *srp; + struct scoutfs_srch_ref ref; + int level; + int ind; + int ret; + + if (WARN_ON_ONCE(at_level < 0 || at_level >= sfl->height)) + return -EINVAL; + + level = sfl->height; + ref = sfl->ref; + while (level--) { + if (ref.blkno == 0) { + ret = -ENOENT; + break; + } + + ret = read_srch_block(sb, wri, level, &ref, &bl); + if (ret < 0) + break; + + if (level == at_level) { + ret = 0; + break; + } + + srp = bl->data; + ind = calc_ref_ind(blk, level); + ref = srp->refs[ind]; + scoutfs_block_put(sb, bl); + bl = NULL; + } + + if (ret < 0) + scoutfs_block_put(sb, bl); + else + *bl_ret = bl; + return ret; +} + +/* + * Walk radix blocks to find the logical file block and return the + * reference to the caller. Flags determine if we cow new dirty blocks, + * allocate new blocks, or return errors for missing blocks (files are + * never sparse, this won't happen). + */ +enum gfb_flags { + GFB_INSERT = (1 << 0), + GFB_DIRTY = (1 << 1), +}; +static int get_file_block(struct super_block *sb, + struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_srch_file *sfl, + int flags, u64 blk, struct scoutfs_block **bl_ret) +{ + struct scoutfs_block *parent = NULL; + struct scoutfs_block_header *hdr; + struct scoutfs_block *bl = NULL; + struct scoutfs_srch_parent *srp; + struct scoutfs_block *new_bl; + struct scoutfs_srch_ref *ref; + u64 blkno = 0; + int level; + int ind; + int err; + int ret; + u8 hei; + + /* see if we need to grow to insert a new largest blk */ + hei = height_for_blk(blk); + while (sfl->height < hei) { + if (!(flags & GFB_INSERT)) { + ret = -ENOENT; + goto out; + } + + ret = scoutfs_alloc_meta(sb, alloc, wri, &blkno); + if (ret < 0) + goto out; + + bl = scoutfs_block_create(sb, blkno); + if (IS_ERR(bl)) { + ret = PTR_ERR(bl); + goto out; + } + blkno = 0; + + scoutfs_block_writer_mark_dirty(sb, wri, bl); + + init_file_block(sb, bl, sfl->height); + if (sfl->height) { + srp = bl->data; + srp->refs[0].blkno = sfl->ref.blkno; + srp->refs[0].seq = sfl->ref.seq; + } + + hdr = bl->data; + sfl->ref.blkno = hdr->blkno; + sfl->ref.seq = hdr->seq; + sfl->height++; + scoutfs_block_put(sb, bl); + bl = NULL; + } + + /* walk file and parent block references to the leaf blocks */ + level = sfl->height; + ref = &sfl->ref; + while (level--) { + /* searching an unused part of the tree */ + if (!ref->blkno && !(flags & GFB_INSERT)) { + ret = -ENOENT; + goto out; + } + + /* read an existing block */ + if (ref->blkno) { + ret = read_srch_block(sb, wri, level, ref, &bl); + if (ret < 0) + goto out; + } + + /* allocate a new block if we need it */ + if (!ref->blkno || ((flags & GFB_DIRTY) && + !scoutfs_block_writer_is_dirty(sb, bl))) { + ret = scoutfs_alloc_meta(sb, alloc, wri, &blkno); + if (ret < 0) + goto out; + + new_bl = scoutfs_block_create(sb, blkno); + if (IS_ERR(new_bl)) { + ret = PTR_ERR(new_bl); + goto out; + } + + if (bl) { + /* cow old block if we have one */ + ret = scoutfs_free_meta(sb, alloc, wri, + bl->blkno); + if (ret) + goto out; + + memcpy(new_bl->data, bl->data, + SCOUTFS_BLOCK_LG_SIZE); + scoutfs_block_put(sb, bl); + bl = new_bl; + hdr = bl->data; + hdr->blkno = cpu_to_le64(bl->blkno); + prandom_bytes(&hdr->seq, sizeof(hdr->seq)); + } else { + /* init new allocated block */ + bl = new_bl; + init_file_block(sb, bl, level); + } + + blkno = 0; + scoutfs_block_writer_mark_dirty(sb, wri, bl); + + /* update file or parent block ref */ + hdr = bl->data; + ref->blkno = hdr->blkno; + ref->seq = hdr->seq; + } + + if (level == 0) { + ret = 0; + break; + } + + srp = bl->data; + ind = calc_ref_ind(blk, level); + ref = &srp->refs[ind]; + + scoutfs_block_put(sb, parent); + parent = bl; + bl = NULL; + } + ret = 0; + +out: + scoutfs_block_put(sb, parent); + + /* return allocated blkno on error */ + if (blkno > 0) { + err = scoutfs_free_meta(sb, alloc, wri, blkno); + BUG_ON(err); /* radix should have been dirty */ + } + + if (ret < 0) { + scoutfs_block_put(sb, bl); + bl = NULL; + } + + /* record that we successfully grew the file */ + if (ret == 0 && (flags & GFB_INSERT) && blk >= le64_to_cpu(sfl->blocks)) + sfl->blocks = cpu_to_le64(blk + 1); + + *bl_ret = bl; + return ret; +} + +int scoutfs_srch_add(struct super_block *sb, + struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_srch_file *sfl, + struct scoutfs_block **bl_ret, + u64 hash, u64 ino, u64 id) +{ + struct scoutfs_srch_block *srb; + struct scoutfs_block *bl = NULL; + u64 blk; + int ret; + struct scoutfs_srch_entry sre = { + .hash = cpu_to_le64(hash), + .ino = cpu_to_le64(ino), + .id = cpu_to_le64(id), + }; + + /* start with a new block or the last existing block */ + if (le64_to_cpu(sfl->blocks) > 1) + blk = le64_to_cpu(sfl->blocks) - 1; + else + blk = 0; + + bl = *bl_ret; +get_last_block: + if (bl == NULL) { + ret = get_file_block(sb, alloc, wri, sfl, + GFB_INSERT | GFB_DIRTY, blk, &bl); + if (ret < 0) { + /* writing into a private file, shouldn't happen */ + WARN_ON_ONCE(ret == -ESTALE); + goto out; + } + } + srb = bl->data; + + /* stop encoding once we might overflow the block */ + if (le32_to_cpu(srb->entry_bytes) > SCOUTFS_SRCH_BLOCK_SAFE_BYTES) { + scoutfs_block_put(sb, bl); + bl = NULL; + blk++; + goto get_last_block; + } + + ret = encode_entry(srb->entries + le32_to_cpu(srb->entry_bytes), + &sre, &srb->tail); + if (ret > 0) { + if (srb->entry_bytes == 0) { + if (blk == 0) { + sfl->first = sre; + sfl->last = sre; + } + srb->first = sre; + srb->last = sre; + } else { + if (sre_cmp(&sre, &sfl->first) < 0) + sfl->first = sre; + else if (sre_cmp(&sre, &sfl->last) > 0) + sfl->last = sre; + if (sre_cmp(&sre, &srb->first) < 0) + srb->first = sre; + else if (sre_cmp(&sre, &srb->last) > 0) + srb->last = sre; + } + srb->tail = sre; + le32_add_cpu(&srb->entry_nr, 1); + le32_add_cpu(&srb->entry_bytes, ret); + le64_add_cpu(&sfl->entries, 1); + ret = 0; + scoutfs_inc_counter(sb, srch_add_entry); + } + +out: + if (ret < 0) { + scoutfs_block_put(sb, bl); + bl = NULL; + } + *bl_ret = bl; + + return ret; +} + +/* + * The caller is dropping an ino/id because the tracking rbtree is full. + * This loses information so we can't return any entries at or after the + * one that we dropped. Update end to the entry before the dropped + * entry if it's less than the current end. + */ +static void set_end_before(struct scoutfs_srch_entry *end, u64 ino, u64 id) +{ + struct scoutfs_srch_entry sre; + + sre.hash = end->hash; + sre.ino = cpu_to_le64(ino); + sre.id = cpu_to_le64(id); + sre_dec(&sre); + if (sre_cmp(&sre, end) < 0) + *end = sre; +} + +/* + * Track an inode and id of an xattr hash that we found while searching. + * We'll return inos from the nodes in order to userspace when we're + * done searching. The first time we see the entry we track it, the + * second time must be a deletion so we remove it. + * + * We count the number of tracked entries here. Once we hit the limit + * we drop entries which are greater than what's tracked. If we track + * new entries which are within the set then we drop the last entry. + * When we drop entries we have to trim the range of entries that we'll + * return because we've lost data. The caller will perform the search + * again from that point, giving them another window of tracked entries + * to fill from that entry. + */ +static int track_found(struct scoutfs_srch_rb_root *sroot, u64 ino, u64 id, + unsigned long limit, struct scoutfs_srch_entry *end) +{ + struct rb_node **node = &sroot->root.rb_node; + struct rb_node *parent = NULL; + struct scoutfs_srch_rb_node *snode; + int cmp = 1; /* set last for first insertion */ + + while (*node) { + parent = *node; + snode = container_of(*node, struct scoutfs_srch_rb_node, node); + + cmp = scoutfs_cmp(ino, snode->ino) ?: + scoutfs_cmp(id, snode->id); + if (cmp < 0) { + node = &(*node)->rb_left; + } else if (cmp > 0) { + node = &(*node)->rb_right; + } else { + /* update last if removed as a dupe */ + if (sroot->last == &snode->node) + sroot->last = rb_prev(sroot->last); + rb_erase(&snode->node, &sroot->root); + kfree(snode); + sroot->nr--; + return 0; + } + } + + /* can't track greater while we're at the limit */ + if (sroot->nr >= limit && cmp > 0 && parent == sroot->last) { + set_end_before(end, ino, id); + return 0; + } + + snode = kzalloc(sizeof(*snode), GFP_NOFS); + if (!snode) + return -ENOMEM; + + rb_link_node(&snode->node, parent, node); + rb_insert_color(&snode->node, &sroot->root); + + /* track a newly inserted last item */ + if (cmp > 0 && parent == sroot->last) + sroot->last = &snode->node; + + snode->ino = ino; + snode->id = id; + sroot->nr++; + + /* remove and update last if we inserted earlier at limit */ + if (sroot->nr > limit && sroot->last != &snode->node) { + snode = container_of(sroot->last, struct scoutfs_srch_rb_node, + node); + sroot->last = rb_prev(sroot->last); + set_end_before(end, snode->ino, snode->id); + rb_erase(&snode->node, &sroot->root); + kfree(snode); + sroot->nr--; + } + + return 0; +} + +/* + * Sweep all the unsorted entries of a log file looking for hash matches + * and tracking their xattr inos and ids. If the tracking sroot fills + * we update end but keep searching because we might find earlier + * entries. + */ +static int search_log_file(struct super_block *sb, + struct scoutfs_srch_file *sfl, + struct scoutfs_srch_rb_root *sroot, + struct scoutfs_srch_entry *start, + struct scoutfs_srch_entry *end, + unsigned long limit) +{ + struct scoutfs_block *bl = NULL; + struct scoutfs_srch_entry sre; + struct scoutfs_srch_entry prev; + struct scoutfs_srch_block *srb; + int ret = 0; + u64 blk; + int pos; + int i; + + for (blk = 0; blk < le64_to_cpu(sfl->blocks); blk++) { + scoutfs_block_put(sb, bl); + ret = get_file_block(sb, NULL, NULL, sfl, 0, blk, &bl); + if (ret < 0) + break; + srb = bl->data; + + memset(&prev, 0, sizeof(prev)); + pos = 0; + scoutfs_inc_counter(sb, srch_search_log_block); + + for (i = 0; i < le32_to_cpu(srb->entry_nr); i++) { + if (pos > SCOUTFS_SRCH_BLOCK_SAFE_BYTES) { + /* can only be inconsistency :/ */ + ret = EIO; + break; + } + + ret = decode_entry(srb->entries + pos, &sre, &prev); + if (ret <= 0) { + /* can only be inconsistency :/ */ + ret = EIO; + break; + } + pos += ret; + prev = sre; + + if (sre_cmp(start, &sre) > 0 || + sre_cmp(&sre, end) > 0) + continue; + + ret = track_found(sroot, le64_to_cpu(sre.ino), + le64_to_cpu(sre.id), limit, end); + if (ret < 0) + break; + } + } + + scoutfs_block_put(sb, bl); + return ret; +} + +/* + * Search a sorted file for entries for inodes that could contain the + * xattr hash that we're looking for. The caller has checked that the + * start entry is contained in the file. We find the first block that + * could contain it and stream entries from there until we fill the + * rbtree or arrive at the end entry. + */ +static int search_sorted_file(struct super_block *sb, + struct scoutfs_srch_file *sfl, + struct scoutfs_srch_rb_root *sroot, + struct scoutfs_srch_entry *start, + struct scoutfs_srch_entry *end, + unsigned long limit) +{ + DECLARE_SRCH_INFO(sb, srinf); + struct scoutfs_srch_block *srb = NULL; + struct scoutfs_srch_entry sre; + struct scoutfs_srch_entry prev; + struct scoutfs_block *bl = NULL; + int ret = 0; + int pos = 0; + s64 left; + s64 right; + u64 first; + u64 blk; + + if (sfl->blocks == 0) + return 0; + + /* binary search for first block in the range */ + first = U64_MAX; + left = 0; + right = le64_to_cpu(sfl->blocks) - 1; + while (left <= right) { + blk = (left + right) >> 1; + + ret = get_file_block(sb, NULL, NULL, sfl, 0, blk, &bl); + if (ret < 0) + goto out; + srb = bl->data; + + if (sre_cmp(end, &srb->first) < 0) { + right = blk - 1; + } else if (sre_cmp(start, &srb->last) > 0) { + left = blk + 1; + } else { + first = min(blk, first); + right = blk - 1; + } + + scoutfs_block_put(sb, bl); + bl = NULL; + } + + /* no blocks in range */ + if (first == U64_MAX) { + ret = 0; + goto out; + } + blk = first; + + /* stream entries until end or we're past the full tracking rb_root */ + for (;;) { + if (bl == NULL) { + /* only check on each new input block */ + if (atomic_read(&srinf->shutdown)) { + ret = -ESHUTDOWN; + goto out; + } + + ret = get_file_block(sb, NULL, NULL, sfl, 0, blk, &bl); + if (ret < 0) + goto out; + srb = bl->data; + + memset(&prev, 0, sizeof(prev)); + pos = 0; + scoutfs_inc_counter(sb, srch_search_sorted_block); + } + + if (pos > SCOUTFS_SRCH_BLOCK_SAFE_BYTES) { + /* can only be inconsistency :/ */ + ret = EIO; + break; + } + + ret = decode_entry(srb->entries + pos, &sre, &prev); + if (ret <= 0) { + /* can only be inconsistency :/ */ + ret = EIO; + break; + } + pos += ret; + prev = sre; + + + if (sre_cmp(start, &sre) > 0) + continue; + if (sre_cmp(&sre, end) > 0) + break; + + ret = track_found(sroot, le64_to_cpu(sre.ino), + le64_to_cpu(sre.id), limit, end); + if (ret < 0) + goto out; + + if (pos >= le32_to_cpu(srb->entry_bytes)) { + scoutfs_block_put(sb, bl); + bl = NULL; + if (++blk == le64_to_cpu(sfl->blocks)) + break; + } + } + ret = 0; +out: + scoutfs_block_put(sb, bl); + return ret; +} + +static int search_file(struct super_block *sb, int type, + struct scoutfs_srch_file *sfl, + struct scoutfs_srch_rb_root *sroot, + struct scoutfs_srch_entry *start, + struct scoutfs_srch_entry *end, unsigned long limit) +{ + + /* ignore files that don't have our hash */ + if (sre_cmp(start, &sfl->last) > 0 || + sre_cmp(end, &sfl->first) < 0) + return 0; + + if (type == SCOUTFS_SRCH_LOG_TYPE) { + scoutfs_inc_counter(sb, srch_search_log); + return search_log_file(sb, sfl, sroot, start, end, limit); + } else { + scoutfs_inc_counter(sb, srch_search_sorted); + return search_sorted_file(sb, sfl, sroot, start, end, limit); + } +} + +static void srch_init_rb_root(struct scoutfs_srch_rb_root *sroot) +{ + sroot->root = RB_ROOT; + sroot->last = NULL; + sroot->nr = 0; +} + +void scoutfs_srch_destroy_rb_root(struct scoutfs_srch_rb_root *sroot) +{ + struct scoutfs_srch_rb_node *snode; + struct scoutfs_srch_rb_node *pos; + + rbtree_postorder_for_each_entry_safe(snode, pos, &sroot->root, node) + kfree(snode); + + srch_init_rb_root(sroot); +} + +/* + * There are no constraints on the distribution of entries in log or + * sorted srch files. We limit the number of entries we track to avoid + * consuming absurd amounts of memory for very large searches. The + * larger the limit the more memory each search will take. The smaller + * this is the more searches will be necessary to find all the entries. + */ +#define SRCH_LIMIT 1000000 + +/* + * Search all the srch files for entries recording that inodes might + * have a given xattr. + * + * Advancing from an inode number that was returned is the only way the + * caller can make forward progress between searches. We might not find + * any inodes if we have the bad luck of pruning all the entries we + * tracked with deletions. We'll restart the search ourselves in this + * case to see if we can find an inode to return to the caller. + */ +int scoutfs_srch_search_xattrs(struct super_block *sb, + struct scoutfs_srch_rb_root *sroot, + u64 hash, u64 ino, u64 last_ino, bool *done) +{ + struct scoutfs_net_roots prev_roots; + struct scoutfs_net_roots roots; + struct scoutfs_srch_entry start; + struct scoutfs_srch_entry end; + struct scoutfs_srch_entry final; + struct scoutfs_log_trees lt; + struct scoutfs_srch_file sfl; + SCOUTFS_BTREE_ITEM_REF(iref); + struct scoutfs_key key; + unsigned long limit = SRCH_LIMIT; + int ret; + + scoutfs_inc_counter(sb, srch_search_xattrs); + + *done = false; + srch_init_rb_root(sroot); + memset(&prev_roots, 0, sizeof(prev_roots)); + + start.hash = cpu_to_le64(hash); + start.ino = cpu_to_le64(ino); + start.id = 0; + final.hash = cpu_to_le64(hash); + final.ino = cpu_to_le64(last_ino); + final.id = cpu_to_le64(U64_MAX); + +retry: + scoutfs_srch_destroy_rb_root(sroot); + + ret = scoutfs_client_get_roots(sb, &roots); + if (ret) + goto out; + memset(&roots.fs_root, 0, sizeof(roots.fs_root)); + + end = final; + + /* search intersecting sorted files, then logs */ + init_srch_key(&key, SCOUTFS_SRCH_BLOCKS_TYPE, 0, 0); + for (;;) { + ret = scoutfs_btree_next(sb, &roots.srch_root, &key, &iref); + if (ret == 0) { + if (iref.key->sk_type != key.sk_type) { + ret = -ENOENT; + } else if (iref.val_len == sizeof(sfl)) { + key = *iref.key; + scoutfs_key_inc(&key); + memcpy(&sfl, iref.val, iref.val_len); + } else { + ret = -EIO; + } + scoutfs_btree_put_iref(&iref); + } + if (ret < 0) { + if (ret == -ENOENT) { + if (key.sk_type == SCOUTFS_SRCH_BLOCKS_TYPE) { + init_srch_key(&key, + SCOUTFS_SRCH_LOG_TYPE, 0, 0); + continue; + } else { + break; + } + } + goto out; + } + + ret = search_file(sb, key.sk_type, &sfl, sroot, + &start, &end, limit); + if (ret < 0) + goto out; + } + + /* search all the log files being written by mounts */ + scoutfs_key_init_log_trees(&key, 0, 0); + for (;;) { + ret = scoutfs_btree_next(sb, &roots.logs_root, &key, &iref); + if (ret == -ENOENT) + break; + if (ret == 0) { + if (iref.val_len == sizeof(lt)) { + key = *iref.key; + scoutfs_key_inc(&key); + memcpy(<, iref.val, iref.val_len); + } else { + ret = -EIO; + } + scoutfs_btree_put_iref(&iref); + } + if (ret < 0) + goto out; + + ret = search_file(sb, SCOUTFS_SRCH_LOG_TYPE, <.srch_file, + sroot, &start, &end, limit); + if (ret < 0) + goto out; + } + + /* keep searching if we didn't find any entries in the limit */ + if (sroot->nr == 0 && sre_cmp(&end, &final) < 0) { + start = end; + sre_inc(&start); + scoutfs_inc_counter(sb, srch_search_retry_empty); + goto retry; + } + + /* let the caller know our search was exhaustive */ + *done = sre_cmp(&end, &final) == 0; + ret = 0; +out: + if (ret == -ESTALE) { + if (memcmp(&prev_roots, &roots, sizeof(roots)) == 0) { + scoutfs_inc_counter(sb, srch_search_stale_eio); + ret = -EIO; + } else { + scoutfs_inc_counter(sb, srch_search_stale_retry); + prev_roots = roots; + goto retry; + } + } + + return ret; +} + +/* + * Running in the server, rotate the client's log file as they commit if + * it's large enough. + */ +int scoutfs_srch_rotate_log(struct super_block *sb, + struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_btree_root *root, + struct scoutfs_srch_file *sfl) +{ + struct scoutfs_key key; + int ret; + + if (le64_to_cpu(sfl->blocks) < SCOUTFS_SRCH_LOG_BLOCK_LIMIT) + return 0; + + init_srch_key(&key, SCOUTFS_SRCH_LOG_TYPE, + le64_to_cpu(sfl->ref.blkno), 0); + ret = scoutfs_btree_insert(sb, alloc, wri, root, &key, + sfl, sizeof(*sfl)); + if (ret == 0) { + memset(sfl, 0, sizeof(*sfl)); + scoutfs_inc_counter(sb, srch_rotate_log); + } + return ret; +} + +/* + * Running in the server, get a compaction operation to send to the + * client. We first see if there are any pending operations to continue + * working on. If not, we see if any tier has enough files waiting for + * a compaction. We first search log files and then each greater size + * tier. We skip input files which are currently being read by busy + * compaction items. + */ +int scoutfs_srch_get_compact(struct super_block *sb, + struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_btree_root *root, + u64 rid, struct scoutfs_srch_compact *sc) +{ + struct scoutfs_srch_file sfl; + SCOUTFS_BTREE_ITEM_REF(iref); + struct scoutfs_spbm busy; + struct scoutfs_key key; + int cur_order = -1; + int order; + int type; + int ret; + int err; + int i; + + /* + * Search for pending or busy items. If we find a pending item + * we move it to busy and return it. We build up a bitmap of + * input files which are in busy items. + */ + scoutfs_spbm_init(&busy); + for (init_srch_key(&key, SCOUTFS_SRCH_PENDING_TYPE, 0, 0); ; + scoutfs_key_inc(&key)) { + + /* _PENDING_ and _BUSY_ are last, _next won't see other types */ + ret = scoutfs_btree_next(sb, root, &key, &iref); + if (ret == -ENOENT) + break; + if (ret == 0) { + if (iref.val_len == sizeof(*sc)) { + key = *iref.key; + memcpy(sc, iref.val, iref.val_len); + } else { + ret = -EIO; + } + scoutfs_btree_put_iref(&iref); + } + if (ret < 0) + goto out; + + /* record all the busy input files */ + if (key.sk_type == SCOUTFS_SRCH_BUSY_TYPE) { + for (i = 0; i < sc->nr; i++) { + ret = scoutfs_spbm_set(&busy, + le64_to_cpu(sc->in[i].sfl.ref.blkno)); + if (ret < 0) + goto out; + } + continue; + } + + /* or move the first pending to busy and return it */ + init_srch_key(&key, SCOUTFS_SRCH_BUSY_TYPE, rid, + le64_to_cpu(sc->id)); + ret = scoutfs_btree_insert(sb, alloc, wri, root, &key, + sc, sizeof(*sc)); + if (ret < 0) + goto out; + + init_srch_key(&key, SCOUTFS_SRCH_PENDING_TYPE, + le64_to_cpu(sc->id), 0); + ret = scoutfs_btree_delete(sb, alloc, wri, root, &key); + if (ret < 0) { + init_srch_key(&key, SCOUTFS_SRCH_BUSY_TYPE, rid, + le64_to_cpu(sc->id)); + err = scoutfs_btree_delete(sb, alloc, wri, root, &key); + BUG_ON(err); /* XXX both pending and busy :/ */ + goto out; + } + + /* found one */ + ret = 0; + goto out; + } + + /* no pending, look for sufficient files to start a new compaction */ + memset(sc, 0, sizeof(struct scoutfs_srch_compact)); + + /* first look for unsorted log files */ + type = SCOUTFS_SRCH_LOG_TYPE; + init_srch_key(&key, type, 0, 0); + + for (;;scoutfs_key_inc(&key)) { + ret = scoutfs_btree_next(sb, root, &key, &iref); + if (ret == -ENOENT) { + ret = 0; + sc->nr = 0; + goto out; + } + + if (ret == 0) { + if (iref.val_len == sizeof(struct scoutfs_srch_file)) { + key = *iref.key; + memcpy(&sfl, iref.val, iref.val_len); + } else { + ret = -EIO; + } + scoutfs_btree_put_iref(&iref); + } + if (ret < 0) + goto out; + + /* skip any files already being compacted */ + if (scoutfs_spbm_test(&busy, le64_to_cpu(sfl.ref.blkno))) + continue; + + /* see if we ran out of log files or files entirely */ + if (key.sk_type != type) { + sc->nr = 0; + if (key.sk_type == SCOUTFS_SRCH_BLOCKS_TYPE) { + type = SCOUTFS_SRCH_BLOCKS_TYPE; + } else { + ret = 0; + goto out; + } + } + + /* reset if we iterated into the next size category */ + if (type == SCOUTFS_SRCH_BLOCKS_TYPE) { + order = fls64(le64_to_cpu(sfl.blocks)) / + SCOUTFS_SRCH_COMPACT_ORDER; + if (order != cur_order) { + cur_order = order; + sc->nr = 0; + } + } + + sc->in[sc->nr++].sfl = sfl; + if (sc->nr == SCOUTFS_SRCH_COMPACT_NR) + break; + + scoutfs_key_inc(&key); + } + + if (type == SCOUTFS_SRCH_LOG_TYPE) + sc->flags = SCOUTFS_SRCH_COMPACT_FLAG_LOG; + else + sc->flags = SCOUTFS_SRCH_COMPACT_FLAG_SORTED; + + /* record that our client has a compaction in process */ + sc->id = sc->in[0].sfl.ref.blkno; + + init_srch_key(&key, SCOUTFS_SRCH_BUSY_TYPE, rid, le64_to_cpu(sc->id)); + ret = scoutfs_btree_insert(sb, alloc, wri, root, &key, + sc, sizeof(*sc)); +out: + scoutfs_spbm_destroy(&busy); + if (ret < 0) + sc->nr = 0; + if (sc->nr < SCOUTFS_SRCH_COMPACT_NR) + memset(&sc->in[sc->nr], 0, + (SCOUTFS_SRCH_COMPACT_NR - sc->nr) * sizeof(sc->in[0])); + return ret; +} + +/* + * get_ previously created a busy item to reserve the files for a compaction. + * The caller has finished the input struct and we can update the persistent + * copy. + */ +int scoutfs_srch_update_compact(struct super_block *sb, + struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_btree_root *root, u64 rid, + struct scoutfs_srch_compact *sc) +{ + struct scoutfs_key key; + + init_srch_key(&key, SCOUTFS_SRCH_BUSY_TYPE, rid, le64_to_cpu(sc->id)); + return scoutfs_btree_update(sb, alloc, wri, root, &key, + sc, sizeof(struct scoutfs_srch_compact)); +} + +static void init_file_key(struct scoutfs_key *key, int type, + struct scoutfs_srch_file *sfl) +{ + if (type == SCOUTFS_SRCH_LOG_TYPE) + init_srch_key(key, type, le64_to_cpu(sfl->ref.blkno), 0); + else + init_srch_key(key, type, le64_to_cpu(sfl->blocks), + le64_to_cpu(sfl->ref.blkno)); +} + +/* + * A compaction has completed so we remove the input file reference + * items and add the output file, if it has contents. If this returns + * an error then the file items were not changed. + */ +static int commit_files(struct super_block *sb, struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_btree_root *root, + struct scoutfs_srch_compact *sc) +{ + struct scoutfs_srch_file *sfl; + struct scoutfs_key key; + int type; + int ret; + int err; + int i; + + if (sc->flags & SCOUTFS_SRCH_COMPACT_FLAG_LOG) + type = SCOUTFS_SRCH_LOG_TYPE; + else + type = SCOUTFS_SRCH_BLOCKS_TYPE; + + if (sc->out.blocks != 0) { + sfl = &sc->out; + init_file_key(&key, SCOUTFS_SRCH_BLOCKS_TYPE, sfl); + ret = scoutfs_btree_insert(sb, alloc, wri, root, &key, + sfl, sizeof(*sfl)); + if (ret < 0) + goto out; + } + + for (i = 0; i < sc->nr; i++) { + sfl = &sc->in[i].sfl; + init_file_key(&key, type, sfl); + + ret = scoutfs_btree_delete(sb, alloc, wri, root, &key); + if (ret < 0) { + while (--i >= 0) { + sfl = &sc->in[i].sfl; + init_file_key(&key, type, sfl); + + err = scoutfs_btree_insert(sb, alloc, wri, + root, &key, + sfl, sizeof(*sfl)); + BUG_ON(err); /* lost srch file */ + } + + if (sc->out.blocks != 0) { + sfl = &sc->out; + init_file_key(&key, SCOUTFS_SRCH_BLOCKS_TYPE, + sfl); + err = scoutfs_btree_delete(sb, alloc, wri, + root, &key); + BUG_ON(err); /* duplicate srch files data */ + } + goto out; + } + } + + ret = 0; +out: + return ret; +} + +/* + * Running in the server: commit the result of a compaction. Given the + * response id, find the compaction's busy item. The busy item is + * returned to a pending item or is advanced depending on the result. + * If the compaction completed then we replace the input files with the + * output files and transition the compaction to delete the input files. + * Once the input files are deleted we can remove the compaction item. + */ +int scoutfs_srch_commit_compact(struct super_block *sb, + struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_btree_root *root, u64 rid, + struct scoutfs_srch_compact *res, + struct scoutfs_alloc_list_head *av, + struct scoutfs_alloc_list_head *fr) +{ + struct scoutfs_srch_compact *pending = NULL; + struct scoutfs_srch_compact *busy; + SCOUTFS_BTREE_ITEM_REF(iref); + struct scoutfs_key key; + int ret; + int err; + int i; + + /* only free allocators when we finish deleting */ + memset(av, 0, sizeof(struct scoutfs_alloc_list_head)); + memset(fr, 0, sizeof(struct scoutfs_alloc_list_head)); + + busy = kzalloc(sizeof(struct scoutfs_srch_compact), GFP_NOFS); + if (busy == NULL) { + ret = -ENOMEM; + goto out; + } + + /* find the record of our compaction */ + init_srch_key(&key, SCOUTFS_SRCH_BUSY_TYPE, rid, le64_to_cpu(res->id)); + ret = scoutfs_btree_lookup(sb, root, &key, &iref); + if (ret == 0) { + if (iref.val_len == sizeof(struct scoutfs_srch_compact)) + memcpy(busy, iref.val, iref.val_len); + else + ret = -EIO; + scoutfs_btree_put_iref(&iref); + } + if (ret < 0) /* XXX leaks allocators */ + goto out; + + /* restore busy to pending if the operation failed */ + if (res->flags & SCOUTFS_SRCH_COMPACT_FLAG_ERROR) { + pending = busy; + ret = 0; + goto update; + } + + /* store result as pending if it isn't done */ + if (!(res->flags & SCOUTFS_SRCH_COMPACT_FLAG_DONE)) { + pending = res; + ret = 0; + goto update; + } + + /* update file references if we finished compaction (!deleting) */ + if (!(res->flags & SCOUTFS_SRCH_COMPACT_FLAG_DELETE)) { + ret = commit_files(sb, alloc, wri, root, res); + if (ret < 0) { + /* XXX we can't commit, shutdown? */ + goto out; + } + + /* transition flags for deleting input files */ + for (i = 0; i < res->nr; i++) { + res->in[i].blk = 0; + res->in[i].pos = 0; + } + res->flags &= ~(SCOUTFS_SRCH_COMPACT_FLAG_DONE | + SCOUTFS_SRCH_COMPACT_FLAG_LOG | + SCOUTFS_SRCH_COMPACT_FLAG_SORTED); + res->flags |= SCOUTFS_SRCH_COMPACT_FLAG_DELETE; + pending = res; + ret = 0; + goto update; + } + + /* ok, finished deleting, reclaim allocs and delete busy */ + *av = res->meta_avail; + *fr = res->meta_freed; + pending = NULL; + ret = 0; +update: + if (pending) { + init_srch_key(&key, SCOUTFS_SRCH_PENDING_TYPE, + le64_to_cpu(pending->id), 0); + ret = scoutfs_btree_insert(sb, alloc, wri, root, &key, + pending, sizeof(*pending)); + if (ret < 0) + goto out; + } + + init_srch_key(&key, SCOUTFS_SRCH_BUSY_TYPE, rid, le64_to_cpu(res->id)); + ret = scoutfs_btree_delete(sb, alloc, wri, root, &key); + if (ret < 0 && pending) { + init_srch_key(&key, SCOUTFS_SRCH_PENDING_TYPE, + le64_to_cpu(pending->id), 0); + err = scoutfs_btree_delete(sb, alloc, wri, root, &key); + BUG_ON(err); /* both busy and pending present */ + } +out: + WARN_ON_ONCE(ret < 0); /* XXX inconsistency */ + kfree(busy); + return ret; +} + +/* + * Remove a busy item for the given client and give the caller its + * allocators. Returns -ENOENT when there are no more items. + */ +int scoutfs_srch_cancel_compact(struct super_block *sb, + struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_btree_root *root, u64 rid, + struct scoutfs_alloc_list_head *av, + struct scoutfs_alloc_list_head *fr) +{ + struct scoutfs_srch_compact *sc; + SCOUTFS_BTREE_ITEM_REF(iref); + struct scoutfs_key key; + struct scoutfs_key last; + int ret; + + init_srch_key(&key, SCOUTFS_SRCH_BUSY_TYPE, rid, 0); + init_srch_key(&last, SCOUTFS_SRCH_BUSY_TYPE, rid, U64_MAX); + + ret = scoutfs_btree_next(sb, root, &key, &iref); + if (ret == 0) { + if (scoutfs_key_compare(iref.key, &last) > 0) { + ret = -ENOENT; + } else if (iref.val_len != sizeof(*sc)) { + ret = -EIO; + } else { + key = *iref.key; + sc = iref.val; + *av = sc->meta_avail; + *fr = sc->meta_freed; + } + scoutfs_btree_put_iref(&iref); + } + if (ret < 0) + goto out; + + ret = scoutfs_btree_delete(sb, alloc, wri, root, &key); +out: + return ret; +} + +/* + * We should commit our progress when we have sufficient dirty blocks or + * don't have enough metadata alloc space for our caller's operations. + */ +static bool should_commit(struct super_block *sb, struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, u32 nr) +{ + return (scoutfs_block_writer_dirty_bytes(sb, wri) >= + SRCH_COMPACT_DIRTY_LIMIT_BYTES) || + scoutfs_alloc_meta_low(sb, alloc, nr); +} + +struct tourn_node { + struct scoutfs_srch_entry sre; + int ind; +}; + +static void tourn_update(struct tourn_node *tnodes, struct tourn_node *tn) +{ + struct tourn_node *sib; + struct tourn_node *par; + size_t ind; + + /* root is at [1] */ + while (tn != &tnodes[1]) { + ind = tn - tnodes; + sib = &tnodes[ind ^ 1]; + par = &tnodes[ind >> 1]; + *par = sre_cmp(&tn->sre, &sib->sre) < 0 ? *tn : *sib; + tn = par; + } +} + +/* return the entry at the current position, can return enoent if done */ +typedef int (*kway_get_t)(struct super_block *sb, + struct scoutfs_srch_entry *sre_ret, void *arg); +/* only called after _get returns 0, advances to next entry for _get */ +typedef void (*kway_advance_t)(struct super_block *sb, void *arg); + +static int kway_merge(struct super_block *sb, + struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_srch_file *sfl, + kway_get_t kway_get, kway_advance_t kway_adv, + void **args, int nr) +{ + DECLARE_SRCH_INFO(sb, srinf); + struct scoutfs_srch_block *srb = NULL; + struct scoutfs_srch_entry last_tail; + struct scoutfs_block *bl = NULL; + struct tourn_node *tnodes; + struct tourn_node *leaves; + struct tourn_node *root; + struct tourn_node *tn; + int last_bytes = 0; + int nr_parents; + int nr_nodes; + int empty = 0; + int ret = 0; + int diff; + u64 blk; + int ind; + int i; + + if (WARN_ON_ONCE(nr <= 1)) + return -EINVAL; + + nr_parents = roundup_pow_of_two(nr) - 1; + /* root at [1] for easy sib/parent index calc, final pad for odd sib */ + nr_nodes = 1 + nr_parents + nr + 1; + tnodes = __vmalloc(nr_nodes * sizeof(struct tourn_node), + GFP_NOFS, PAGE_KERNEL); + if (!tnodes) + return -ENOMEM; + + memset(tnodes, 0xff, nr_nodes * sizeof(struct tourn_node)); + root = &tnodes[1]; + leaves = &root[nr_parents]; + + /* initialize tournament leaves */ + for (i = 0; i < nr; i++) { + tn = &leaves[i]; + tn->ind = i; + ret = kway_get(sb, &tn->sre, args[i]); + if (ret == 0) { + tourn_update(tnodes, &leaves[i]); + } else if (ret == -ENOENT) { + empty++; + } else { + goto out; + } + } + + /* always append new blocks */ + blk = le64_to_cpu(sfl->blocks); + while (empty < nr) { + if (bl == NULL) { + if (atomic_read(&srinf->shutdown)) { + ret = -ESHUTDOWN; + goto out; + } + + /* could grow and dirty to a leaf */ + if (should_commit(sb, alloc, wri, sfl->height + 1)) { + ret = 0; + goto out; + } + + ret = get_file_block(sb, alloc, wri, sfl, + GFB_INSERT | GFB_DIRTY, blk, &bl); + if (ret < 0) + goto out; + srb = bl->data; + scoutfs_inc_counter(sb, srch_compact_dirty_block); + } + + if (sre_cmp(&root->sre, &srb->last) != 0) { + last_bytes = le32_to_cpu(srb->entry_bytes); + last_tail = srb->last; + ret = encode_entry(srb->entries + + le32_to_cpu(srb->entry_bytes), + &root->sre, &srb->tail); + if (WARN_ON_ONCE(ret <= 0)) { + /* shouldn't happen */ + ret = -EIO; + goto out; + } + + if (srb->entry_bytes == 0) { + if (blk == 0) + sfl->first = root->sre; + srb->first = root->sre; + } + le32_add_cpu(&srb->entry_nr, 1); + le32_add_cpu(&srb->entry_bytes, ret); + srb->last = root->sre; + srb->tail = root->sre; + sfl->last = root->sre; + le64_add_cpu(&sfl->entries, 1); + ret = 0; + + if (le32_to_cpu(srb->entry_bytes) > + SCOUTFS_SRCH_BLOCK_SAFE_BYTES) { + scoutfs_block_put(sb, bl); + bl = NULL; + blk++; + } + + scoutfs_inc_counter(sb, srch_compact_entry); + + } else { + /* + * Duplicate entries indicate deletion so we + * undo the previously encoded entry and ignore + * this entry. This only happens within each + * block. Deletions can span block boundaries + * and will be filtered out by search and + * hopefully removed in future compactions. + */ + diff = le32_to_cpu(srb->entry_bytes) - last_bytes; + if (diff) { + memset(srb->entries + last_bytes, 0, diff); + if (srb->entry_bytes == 0) { + /* last_tail will be 0 */ + if (blk == 0) + sfl->first = last_tail; + srb->first = last_tail; + } + le32_add_cpu(&srb->entry_nr, -1); + srb->entry_bytes = cpu_to_le32(last_bytes); + srb->last = last_tail; + srb->tail = last_tail; + sfl->last = last_tail; + le64_add_cpu(&sfl->entries, -1); + } + + scoutfs_inc_counter(sb, srch_compact_removed_entry); + } + + /* get the next */ + ind = root->ind; + tn = &leaves[ind]; + kway_adv(sb, args[ind]); + ret = kway_get(sb, &tn->sre, args[ind]); + if (ret == -ENOENT) { + /* this index is done */ + memset(&tn->sre, 0xff, sizeof(tn->sre)); + empty++; + ret = 0; + } else if (ret < 0) { + goto out; + } + + /* update the tourney and carry on */ + tourn_update(tnodes, tn); +#if 0 + /* would be worth it if we have uneven key distribution */ + if (ind < nr - 1) { + /* order doesn't matter, fill hole */ + swap(args[ind], args[nr - 1]); + swap(tn->sre, leaves[nr - 1].sre); + } + /* drop a level of the tree when we shrink to a power of 2 */ + if (nr > 0 && is_power_of_two(nr)) { + memcpy(leaves - nr, leaves, nr * sizeof(*tn)); + leaves -= nr; + for (i = 0; i < nr; i += 2) + tourn_update(least, leaves[i]); + } +#endif + } + + /* could stream a final index.. arguably a small portion of work */ + +out: + scoutfs_block_put(sb, bl); + vfree(tnodes); + return ret; +} + +#define SRES_PER_PAGE (PAGE_SIZE / sizeof(struct scoutfs_srch_entry)) + +static struct scoutfs_srch_entry *page_priv_sre(struct page *page) +{ + return (struct scoutfs_srch_entry *)page_address(page) + page->private; +} + +static int kway_get_page(struct super_block *sb, + struct scoutfs_srch_entry *sre_ret, void *arg) +{ + struct page *page = arg; + struct scoutfs_srch_entry *sre = page_priv_sre(page); + + if (page->private >= SRES_PER_PAGE || sre->ino == 0) + return -ENOENT; + + *sre_ret = *sre; + return 0; +} + +static void kway_adv_page(struct super_block *sb, void *arg) +{ + struct page *page = arg; + + page->private++; +} + +static int cmp_page_sre(const void *A, const void *B) +{ + const struct scoutfs_srch_entry *a = A; + const struct scoutfs_srch_entry *b = B; + + return sre_cmp(a, b); +} + +static void swap_page_sre(void *A, void *B, int size) +{ + struct scoutfs_srch_entry *a = A; + struct scoutfs_srch_entry *b = B; + + swap(*a, *b); +} + +/* + * Compact a set of log files by sorting all their entries and writing + * them to a sorted output file. We decode all the file's entries into + * pages, sort the contents of each page, and then stream a k-way merge + * of the entries in the pages into an output file. While not sorted, + * the input log files entries are encoded so we can allocate quite a + * bit more memory in pages than the files took in blocks on disk (~2x + * typically, ~10x worst case). + * + * Because we read and sort all the input files we must perform the full + * compaction in one operation. The server must have given us a + * sufficiently large avail/freed lists, otherwise we'll return ENOSPC. + */ +static int compact_logs(struct super_block *sb, + struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_srch_compact *sc) +{ + DECLARE_SRCH_INFO(sb, srinf); + struct scoutfs_srch_block *srb = NULL; + struct scoutfs_srch_entry *sre; + struct scoutfs_srch_entry prev; + struct scoutfs_block *bl = NULL; + struct scoutfs_srch_file *sfl; + struct page *page = NULL; + struct page *tmp; + void **args = NULL; + int nr_pages = 0; + LIST_HEAD(pages); + int sfl_ind; + u64 blk = 0; + int pos = 0; + int ret; + int i; + + if (sc->nr <= 1) { + ret = -EINVAL; + goto out; + } + + memset(&prev, 0, sizeof(prev)); + + /* decode all the log file's block's entries into pages */ + for (sfl_ind = 0, sfl = &sc->in[0].sfl; sfl_ind < sc->nr; ) { + + if (bl == NULL) { + /* only check on each new input block */ + if (atomic_read(&srinf->shutdown)) { + ret = -ESHUTDOWN; + goto out; + } + + ret = get_file_block(sb, NULL, NULL, sfl, 0, blk, &bl); + if (ret < 0) + goto out; + srb = bl->data; + } + + if (page == NULL) { + page = alloc_page(GFP_NOFS); + if (!page) { + ret = -ENOMEM; + goto out; + } + page->private = 0; + list_add_tail(&page->list, &pages); + nr_pages++; + scoutfs_inc_counter(sb, srch_compact_log_page); + } + + sre = page_priv_sre(page); + + if (pos > SCOUTFS_SRCH_BLOCK_SAFE_BYTES) { + /* can only be inconsistency :/ */ + ret = EIO; + break; + } + + ret = decode_entry(srb->entries + pos, sre, &prev); + if (ret <= 0) { + /* can only be inconsistency :/ */ + ret = EIO; + goto out; + } + prev = *sre; + + pos += ret; + if (pos >= le32_to_cpu(srb->entry_bytes)) { + scoutfs_block_put(sb, bl); + bl = NULL; + memset(&prev, 0, sizeof(prev)); + pos = 0; + if (++blk == le64_to_cpu(sfl->blocks)) { + blk = 0; + sfl_ind++; + sfl = &sc->in[sfl_ind].sfl; + } + } + + if (++page->private == SRES_PER_PAGE) + page = NULL; + } + + /* add a terminal entry to the last partial page */ + if (page) { + sre = page_priv_sre(page); + sre->ino = 0; + } + + /* allocate args array for k-way merge */ + args = vmalloc(nr_pages * sizeof(struct page *)); + if (!args) { + ret = -ENOMEM; + goto out; + } + + /* sort page entries and reset private for _next */ + i = 0; + list_for_each_entry(page, &pages, list) { + args[i++] = page; + + if (atomic_read(&srinf->shutdown)) { + ret = -ESHUTDOWN; + goto out; + } + + sort(page_address(page), page->private, + sizeof(struct scoutfs_srch_entry), cmp_page_sre, + swap_page_sre); + page->private = 0; + + } + + ret = kway_merge(sb, alloc, wri, &sc->out, kway_get_page, kway_adv_page, + args, nr_pages); + if (ret < 0) + goto out; + + /* make sure we finished all the pages */ + list_for_each_entry(page, &pages, list) { + sre = page_priv_sre(page); + if (page->private < SRES_PER_PAGE && sre->ino != 0) { + ret = -ENOSPC; + goto out; + } + } + + sc->flags |= SCOUTFS_SRCH_COMPACT_FLAG_DONE; + ret = 0; +out: + scoutfs_block_put(sb, bl); + vfree(args); + list_for_each_entry_safe(page, tmp, &pages, list) { + list_del(&page->list); + __free_page(page); + } + + return ret; +} + +struct kway_file_reader { + struct scoutfs_srch_file *sfl; + struct scoutfs_block *bl; + struct scoutfs_srch_entry prev; + struct scoutfs_srch_entry decoded_sre; + u64 blk; + u32 skip; + u32 pos; + int decoded_bytes; +}; + +static int kway_get_reader(struct super_block *sb, + struct scoutfs_srch_entry *sre_ret, void *arg) +{ + struct kway_file_reader *rdr = arg; + struct scoutfs_srch_block *srb; + int ret; + + if (rdr->blk == le64_to_cpu(rdr->sfl->blocks)) + return -ENOENT; + + if (rdr->bl == NULL) { + ret = get_file_block(sb, NULL, NULL, rdr->sfl, 0, rdr->blk, + &rdr->bl); + if (ret < 0) + return ret; + + memset(&rdr->prev, 0, sizeof(rdr->prev)); + } + srb = rdr->bl->data; + + if (rdr->pos > SCOUTFS_SRCH_BLOCK_SAFE_BYTES || + rdr->skip >= SCOUTFS_SRCH_BLOCK_SAFE_BYTES || + rdr->skip >= le32_to_cpu(srb->entry_bytes)) { + /* XXX inconsistency */ + return -EIO; + } + + /* decode entry, possibly skipping start of the block */ + while (rdr->decoded_bytes == 0 || rdr->pos < rdr->skip) { + ret = decode_entry(srb->entries + rdr->pos, + &rdr->decoded_sre, &rdr->prev); + if (ret <= 0) { + /* XXX inconsistency */ + return -EIO; + } + + rdr->decoded_bytes = ret; + + if (rdr->pos < rdr->skip) { + rdr->prev = rdr->decoded_sre; + rdr->pos += ret; + if (rdr->pos >= rdr->skip) + rdr->skip = 0; + rdr->decoded_bytes = 0; + } + } + + *sre_ret = rdr->decoded_sre; + return 0; +} + +static void kway_adv_reader(struct super_block *sb, void *arg) +{ + struct kway_file_reader *rdr = arg; + struct scoutfs_srch_block *srb; + + /* _get must have set */ + BUG_ON(rdr->bl == NULL); + BUG_ON(rdr->decoded_bytes == 0); + + rdr->prev = rdr->decoded_sre; + rdr->pos += rdr->decoded_bytes; + rdr->decoded_bytes = 0; + + srb = rdr->bl->data; + if (rdr->pos >= le32_to_cpu(srb->entry_bytes)) { + rdr->pos = 0; + scoutfs_block_put(sb, rdr->bl); + rdr->bl = NULL; + rdr->blk++; + } +} + +/* + * Compact a set of sorted files by performing a k-way merge of the files + * into an output sorted file. The k-way merge works with an iterator + * which reads blocks and decodes entries. + */ +static int compact_sorted(struct super_block *sb, + struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_srch_compact *sc) +{ + struct kway_file_reader *rdrs = NULL; + void **args = NULL; + int ret; + int nr; + int i; + + if (WARN_ON_ONCE(sc->nr <= 1)) + return -EINVAL; + + nr = sc->nr; + + /* allocate args array for k-way merge */ + rdrs = kmalloc_array(nr, sizeof(rdrs[0]), __GFP_ZERO | GFP_NOFS); + args = kmalloc_array(nr, sizeof(args[0]), GFP_NOFS); + if (!rdrs || !args) { + ret = -ENOMEM; + goto out; + } + + for (i = 0; i < nr; i++) { + if (le64_to_cpu(sc->in[i].blk) > + le64_to_cpu(sc->in[i].sfl.blocks)) { + ret = -EINVAL; + goto out; + } + + rdrs[i].sfl = &sc->in[i].sfl; + rdrs[i].blk = le64_to_cpu(sc->in[i].blk); + rdrs[i].skip = le64_to_cpu(sc->in[i].pos); + args[i] = &rdrs[i]; + } + + ret = kway_merge(sb, alloc, wri, &sc->out, kway_get_reader, + kway_adv_reader, args, nr); + + sc->flags |= SCOUTFS_SRCH_COMPACT_FLAG_DONE; + for (i = 0; i < nr; i++) { + sc->in[i].blk = cpu_to_le64(rdrs[i].blk); + sc->in[i].pos = cpu_to_le64(rdrs[i].pos); + + if (rdrs[i].blk < le64_to_cpu(sc->in[i].sfl.blocks)) + sc->flags &= ~SCOUTFS_SRCH_COMPACT_FLAG_DONE; + } +out: + for (i = 0; rdrs && i < nr; i++) + scoutfs_block_put(sb, rdrs[i].bl); + kfree(rdrs); + kfree(args); + + return ret; +} + +/* + * Delete a file that has been compacted and is no longer referenced by + * items in the srch_root. The server protects the input file from + * other compactions while we're working, but other readers could be + * still trying to read it while searching. + * + * We don't modify the blocks to avoid the cost of allocating and + * freeing dirty parent metadata blocks, and we want to avoid triggering + * stale reads in racing readers. We free blocks from leaf parents + * upwards and from left to right. Once we've freed a block we never + * visit it again. We store our walk position in each file's compact + * input so that it can be stored in pending items as progress is made + * over multiple operations. + */ +static int delete_file(struct super_block *sb, struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_srch_compact_input *in) +{ + struct scoutfs_block *bl = NULL; + struct scoutfs_srch_parent *srp; + u64 blkno; + u64 blk; + u64 inc; + int level; + int ret; + int i; + + blk = le64_to_cpu(in->blk); + level = max(le64_to_cpu(in->pos), 1ULL); + + if (level > in->sfl.height) { + ret = 0; + goto out; + } + + for (; level < in->sfl.height; level++) { + + for (inc = 1, i = 2; i <= level; i++) + inc *= SCOUTFS_SRCH_PARENT_REFS; + + while (blk < le64_to_cpu(in->sfl.blocks)) { + + ret = read_path_block(sb, wri, &in->sfl, blk, level, + &bl); + if (ret < 0) + goto out; + srp = bl->data; + + for (i = calc_ref_ind(blk, level); + i < SCOUTFS_SRCH_PARENT_REFS && + blk < le64_to_cpu(in->sfl.blocks); + i++, blk += inc) { + + blkno = le64_to_cpu(srp->refs[i].blkno); + if (!blkno) + continue; + + /* free below, then final root block */ + if (should_commit(sb, alloc, wri, 2)) { + ret = 0; + goto out; + } + + ret = scoutfs_free_meta(sb, alloc, wri, blkno); + if (ret < 0) + goto out; + } + + scoutfs_block_put(sb, bl); + bl = NULL; + } + blk = 0; + } + + if (level == in->sfl.height) { + ret = scoutfs_free_meta(sb, alloc, wri, + le64_to_cpu(in->sfl.ref.blkno)); + if (ret < 0) + goto out; + level++; + } + + ret = 0; +out: + in->blk = cpu_to_le64(blk); + in->pos = cpu_to_le64(level); + + scoutfs_block_put(sb, bl); + return ret; +} + +static int delete_files(struct super_block *sb, struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_srch_compact *sc) +{ + int ret; + int i; + + for (i = 0; i < sc->nr; i++) { + ret = delete_file(sb, alloc, wri, &sc->in[i]); + if (ret < 0 || + (le64_to_cpu(sc->in[i].pos) <= sc->in[i].sfl.height)) + break; + } + if (i == sc->nr) + sc->flags |= SCOUTFS_SRCH_COMPACT_FLAG_DONE; + + return ret; +} + +/* wait 10s between compact attempts on error, immediate after success */ +#define SRCH_COMPACT_DELAY_MS (10 * MSEC_PER_SEC) + +/* + * Get a compaction operation from the server, sort the entries from the + * input files as they're read, and stream the remaining sorted entries + * into a newly written output file. The server is protecting the input + * files from other compactions, they will be stable. The server gives + * us a populated allocator that should be enough to write a new file + * and delete the old file blocks. We'll regularly write out dirty + * blocks as we hit a dirty limit threshold so there will be some cow + * overhead of repeatedly dirtying, say, parent allocator and file radix + * blocks. We don't reclaim freed blocks in the allocator after each + * write so the initial allocator pool has to account for that cow + * overhead. + * + * All of our modifications are written into free blocks from the + * filesystem's perspective. If anything goes wrong we return an error + * and the server will ignore all our work and reclaim the initial + * allocator they gave us. + */ +static void scoutfs_srch_compact_worker(struct work_struct *work) +{ + struct srch_info *srinf = container_of(work, struct srch_info, + compact_dwork.work); + struct scoutfs_srch_compact *sc = NULL; + struct super_block *sb = srinf->sb; + struct scoutfs_block_writer wri; + struct scoutfs_alloc alloc; + unsigned long delay; + int ret; + + sc = kmalloc(sizeof(struct scoutfs_srch_compact), GFP_NOFS); + if (sc == NULL) { + ret = -ENOMEM; + goto out; + } + + scoutfs_block_writer_init(sb, &wri); + + ret = scoutfs_client_srch_get_compact(sb, sc); + if (ret < 0 || sc->nr == 0) + goto out; + + scoutfs_alloc_init(&alloc, &sc->meta_avail, &sc->meta_freed); + + if (sc->flags & SCOUTFS_SRCH_COMPACT_FLAG_LOG) { + ret = compact_logs(sb, &alloc, &wri, sc); + + } else if (sc->flags & SCOUTFS_SRCH_COMPACT_FLAG_SORTED) { + ret = compact_sorted(sb, &alloc, &wri, sc); + + } else if (sc->flags & SCOUTFS_SRCH_COMPACT_FLAG_DELETE) { + ret = delete_files(sb, &alloc, &wri, sc); + + } else { + ret = -EINVAL; + } + if (ret < 0) + goto commit; + + ret = scoutfs_block_writer_write(sb, &wri); +commit: + /* the server won't use our partial compact if _ERROR is set */ + sc->meta_avail = alloc.avail; + sc->meta_freed = alloc.freed; + sc->flags |= ret < 0 ? SCOUTFS_SRCH_COMPACT_FLAG_ERROR : 0; + + ret = scoutfs_client_srch_commit_compact(sb, sc); +out: + /* our allocators and files should be stable */ + WARN_ON_ONCE(ret == -ESTALE); + + scoutfs_block_writer_forget_all(sb, &wri); + if (!atomic_read(&srinf->shutdown)) { + delay = ret == 0 ? 0 : msecs_to_jiffies(SRCH_COMPACT_DELAY_MS); + queue_delayed_work(srinf->workq, &srinf->compact_dwork, delay); + } + + kfree(sc); +} + +void scoutfs_srch_destroy(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + DECLARE_SRCH_INFO(sb, srinf); + + if (!srinf) + return; + + if (srinf->workq) { + /* pending grace work queues normal work */ + atomic_set(&srinf->shutdown, 1); + cancel_delayed_work_sync(&srinf->compact_dwork); + flush_workqueue(srinf->workq); + destroy_workqueue(srinf->workq); + } + + kfree(srinf); + sbi->srch_info = NULL; +} + +int scoutfs_srch_setup(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct srch_info *srinf; + int ret; + + srinf = kzalloc(sizeof(struct srch_info), GFP_KERNEL); + if (!srinf) + return -ENOMEM; + + srinf->sb = sb; + atomic_set(&srinf->shutdown, 0); + INIT_DELAYED_WORK(&srinf->compact_dwork, scoutfs_srch_compact_worker); + sbi->srch_info = srinf; + + srinf->workq = alloc_workqueue("scoutfs_srch_compact", + WQ_NON_REENTRANT | WQ_UNBOUND | + WQ_HIGHPRI, 0); + if (!srinf->workq) { + ret = -ENOMEM; + goto out; + } + + queue_delayed_work(srinf->workq, &srinf->compact_dwork, + msecs_to_jiffies(SRCH_COMPACT_DELAY_MS)); + + ret = 0; +out: + if (ret) + scoutfs_srch_destroy(sb); + + return ret; +} diff --git a/kmod/src/srch.h b/kmod/src/srch.h new file mode 100644 index 00000000..69448ab3 --- /dev/null +++ b/kmod/src/srch.h @@ -0,0 +1,68 @@ +#ifndef _SCOUTFS_SRCH_H_ +#define _SCOUTFS_SRCH_H_ + +struct scoutfs_block; + +struct scoutfs_srch_rb_root { + struct rb_root root; + struct rb_node *last; + unsigned long nr; +}; + +struct scoutfs_srch_rb_node { + struct rb_node node; + u64 ino; + u64 id; +}; + +#define scoutfs_srch_foreach_rb_node(snode, node, sroot) \ + for (node = rb_first(&(sroot)->root); \ + node && (snode = container_of(node, struct scoutfs_srch_rb_node, \ + node), 1); \ + node = rb_next(node)) + +int scoutfs_srch_add(struct super_block *sb, + struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_srch_file *sfl, + struct scoutfs_block **bl_ret, + u64 hash, u64 ino, u64 id); + +void scoutfs_srch_destroy_rb_root(struct scoutfs_srch_rb_root *sroot); +int scoutfs_srch_search_xattrs(struct super_block *sb, + struct scoutfs_srch_rb_root *sroot, + u64 hash, u64 ino, u64 last_ino, bool *done); + +int scoutfs_srch_rotate_log(struct super_block *sb, + struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_btree_root *root, + struct scoutfs_srch_file *sfl); +int scoutfs_srch_get_compact(struct super_block *sb, + struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_btree_root *root, + u64 rid, struct scoutfs_srch_compact *sc); +int scoutfs_srch_update_compact(struct super_block *sb, + struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_btree_root *root, u64 rid, + struct scoutfs_srch_compact *sc); +int scoutfs_srch_commit_compact(struct super_block *sb, + struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_btree_root *root, u64 rid, + struct scoutfs_srch_compact *res, + struct scoutfs_alloc_list_head *av, + struct scoutfs_alloc_list_head *fr); +int scoutfs_srch_cancel_compact(struct super_block *sb, + struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_btree_root *root, u64 rid, + struct scoutfs_alloc_list_head *av, + struct scoutfs_alloc_list_head *fr); + +void scoutfs_srch_destroy(struct super_block *sb); +int scoutfs_srch_setup(struct super_block *sb); + +#endif diff --git a/kmod/src/super.c b/kmod/src/super.c new file mode 100644 index 00000000..926531d6 --- /dev/null +++ b/kmod/src/super.c @@ -0,0 +1,716 @@ +/* + * Copyright (C) 2015 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "super.h" +#include "block.h" +#include "export.h" +#include "format.h" +#include "inode.h" +#include "dir.h" +#include "msg.h" +#include "counters.h" +#include "triggers.h" +#include "trans.h" +#include "data.h" +#include "lock.h" +#include "net.h" +#include "client.h" +#include "server.h" +#include "options.h" +#include "sysfs.h" +#include "quorum.h" +#include "forest.h" +#include "srch.h" +#include "item.h" +#include "alloc.h" +#include "scoutfs_trace.h" + +static struct dentry *scoutfs_debugfs_root; + +static DEFINE_PER_CPU(u64, clock_sync_ids) = 0; + +/* + * Give the caller a unique clock sync id for a message they're about to + * send. We make the ids reasonably globally unique by using randomly + * initialized per-cpu 64bit counters. + */ +__le64 scoutfs_clock_sync_id(void) +{ + u64 rnd = 0; + u64 ret; + u64 *id; + +retry: + preempt_disable(); + id = this_cpu_ptr(&clock_sync_ids); + if (*id == 0) { + if (rnd == 0) { + preempt_enable(); + get_random_bytes(&rnd, sizeof(rnd)); + goto retry; + } + *id = rnd; + } + + ret = ++(*id); + preempt_enable(); + + return cpu_to_le64(ret); +} + +struct statfs_free_blocks { + u64 meta; + u64 data; +}; + +static int count_free_blocks(struct super_block *sb, void *arg, int owner, + u64 id, bool meta, bool avail, u64 blocks) +{ + struct statfs_free_blocks *sfb = arg; + + if (meta) + sfb->meta += blocks; + else + sfb->data += blocks; + + return 0; +} + +/* + * Build the free block counts by having alloc read all the persistent + * blocks which contain allocators and calling us for each of them. + * Only the super block reads aren't cached so repeatedly calling statfs + * is like repeated O_DIRECT IO. We can add a cache and stale results + * if that IO becomes a problem. + * + * We fake the number of free inodes value by assuming that we can fill + * free blocks with a certain number of inodes. We then the number of + * current inodes to that free count to determine the total possible + * inodes. + * + * The fsid that we report is constructed from the xor of the first two + * and second two little endian u32s that make up the uuid bytes. + */ +static int scoutfs_statfs(struct dentry *dentry, struct kstatfs *kst) +{ + struct super_block *sb = dentry->d_inode->i_sb; + struct scoutfs_super_block *super = NULL; + struct statfs_free_blocks sfb = {0,}; + __le32 uuid[4]; + int ret; + + scoutfs_inc_counter(sb, statfs); + + super = kzalloc(sizeof(struct scoutfs_super_block), GFP_NOFS); + if (!super) { + ret = -ENOMEM; + goto out; + } + + ret = scoutfs_read_super(sb, super); + if (ret) + goto out; + + ret = scoutfs_alloc_foreach(sb, count_free_blocks, &sfb); + if (ret < 0) + goto out; + + kst->f_bfree = (sfb.meta << SCOUTFS_BLOCK_SM_LG_SHIFT) + sfb.data; + kst->f_type = SCOUTFS_SUPER_MAGIC; + kst->f_bsize = SCOUTFS_BLOCK_SM_SIZE; + kst->f_blocks = (le64_to_cpu(super->total_meta_blocks) << + SCOUTFS_BLOCK_SM_LG_SHIFT) + + le64_to_cpu(super->total_data_blocks); + kst->f_bavail = kst->f_bfree; + + /* arbitrarily assume ~1K / empty file */ + kst->f_ffree = sfb.meta * (SCOUTFS_BLOCK_LG_SIZE / 1024); + kst->f_files = kst->f_ffree + le64_to_cpu(super->next_ino); + + BUILD_BUG_ON(sizeof(uuid) != sizeof(super->uuid)); + memcpy(uuid, super->uuid, sizeof(uuid)); + kst->f_fsid.val[0] = le32_to_cpu(uuid[0]) ^ le32_to_cpu(uuid[1]); + kst->f_fsid.val[1] = le32_to_cpu(uuid[2]) ^ le32_to_cpu(uuid[3]); + kst->f_namelen = SCOUTFS_NAME_LEN; + kst->f_frsize = SCOUTFS_BLOCK_SM_SIZE; + + /* the vfs fills f_flags */ + ret = 0; +out: + kfree(super); + + /* + * We don't take cluster locks in statfs which makes it a very + * convenient place to trigger lock reclaim for debugging. We + * try to free as many locks as possible. + */ + if (scoutfs_trigger(sb, STATFS_LOCK_PURGE)) + scoutfs_free_unused_locks(sb, -1UL); + + return ret; +} + +static int scoutfs_show_options(struct seq_file *seq, struct dentry *root) +{ + struct super_block *sb = root->d_sb; + struct mount_options *opts = &SCOUTFS_SB(sb)->opts; + + seq_printf(seq, ",server_addr="SIN_FMT, SIN_ARG(&opts->server_addr)); + seq_printf(seq, ",metadev_path=%s", opts->metadev_path); + + return 0; +} + +static ssize_t metadev_path_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct super_block *sb = SCOUTFS_SYSFS_ATTRS_SB(kobj); + struct mount_options *opts = &SCOUTFS_SB(sb)->opts; + + return snprintf(buf, PAGE_SIZE, "%s", opts->metadev_path); +} +SCOUTFS_ATTR_RO(metadev_path); + +static ssize_t server_addr_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct super_block *sb = SCOUTFS_SYSFS_ATTRS_SB(kobj); + struct mount_options *opts = &SCOUTFS_SB(sb)->opts; + + return snprintf(buf, PAGE_SIZE, SIN_FMT"\n", + SIN_ARG(&opts->server_addr)); +} +SCOUTFS_ATTR_RO(server_addr); + +static struct attribute *mount_options_attrs[] = { + SCOUTFS_ATTR_PTR(metadev_path), + SCOUTFS_ATTR_PTR(server_addr), + NULL, +}; + +static int scoutfs_sync_fs(struct super_block *sb, int wait) +{ + trace_scoutfs_sync_fs(sb, wait); + scoutfs_inc_counter(sb, trans_commit_sync_fs); + + return scoutfs_trans_sync(sb, wait); +} + +/* + * Data dev is closed by generic code, but we have to explicitly close the meta + * dev. + */ +static void scoutfs_metadev_close(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + + if (sbi->meta_bdev) { + blkdev_put(sbi->meta_bdev, SCOUTFS_META_BDEV_MODE); + sbi->meta_bdev = NULL; + } +} + +/* + * This destroys all the state that's built up in the sb info during + * mount. It's called by us on errors during mount if we haven't set + * s_root, by mount after returning errors if we have set s_root, and by + * unmount after having synced the super. + */ +static void scoutfs_put_super(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + + trace_scoutfs_put_super(sb); + + sbi->shutdown = true; + + scoutfs_data_destroy(sb); + scoutfs_srch_destroy(sb); + + scoutfs_unlock(sb, sbi->rid_lock, SCOUTFS_LOCK_WRITE); + sbi->rid_lock = NULL; + + scoutfs_shutdown_trans(sb); + scoutfs_client_destroy(sb); + scoutfs_inode_destroy(sb); + scoutfs_item_destroy(sb); + scoutfs_forest_destroy(sb); + + /* the server locks the listen address and compacts */ + scoutfs_lock_shutdown(sb); + scoutfs_server_destroy(sb); + scoutfs_net_destroy(sb); + scoutfs_lock_destroy(sb); + + /* server clears quorum leader flag during shutdown */ + scoutfs_quorum_destroy(sb); + + scoutfs_block_destroy(sb); + scoutfs_destroy_triggers(sb); + scoutfs_options_destroy(sb); + scoutfs_sysfs_destroy_attrs(sb, &sbi->mopts_ssa); + debugfs_remove(sbi->debug_root); + scoutfs_destroy_counters(sb); + scoutfs_destroy_sysfs(sb); + scoutfs_metadev_close(sb); + + kfree(sbi->opts.metadev_path); + kfree(sbi); + + sb->s_fs_info = NULL; +} + +static const struct super_operations scoutfs_super_ops = { + .alloc_inode = scoutfs_alloc_inode, + .drop_inode = scoutfs_drop_inode, + .evict_inode = scoutfs_evict_inode, + .destroy_inode = scoutfs_destroy_inode, + .sync_fs = scoutfs_sync_fs, + .statfs = scoutfs_statfs, + .show_options = scoutfs_show_options, + .put_super = scoutfs_put_super, +}; + +/* + * Write the caller's super. The caller has always read a valid super + * before modifying and writing it. The caller's super is modified + * to reflect the write. + */ +int scoutfs_write_super(struct super_block *sb, + struct scoutfs_super_block *super) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + + le64_add_cpu(&super->hdr.seq, 1); + + return scoutfs_block_write_sm(sb, sbi->meta_bdev, SCOUTFS_SUPER_BLKNO, + &super->hdr, + sizeof(struct scoutfs_super_block)); +} + +/* + * Read super, specifying bdev. + */ +static int scoutfs_read_super_from_bdev(struct super_block *sb, + struct block_device *bdev, + struct scoutfs_super_block *super_res) +{ + struct scoutfs_super_block *super; + __le32 calc; + u64 blkno; + int ret; + + super = kmalloc(sizeof(struct scoutfs_super_block), GFP_NOFS); + if (!super) + return -ENOMEM; + + ret = scoutfs_block_read_sm(sb, bdev, SCOUTFS_SUPER_BLKNO, &super->hdr, + sizeof(struct scoutfs_super_block), &calc); + if (ret < 0) + goto out; + + if (super->hdr.magic != cpu_to_le32(SCOUTFS_BLOCK_MAGIC_SUPER)) { + scoutfs_err(sb, "super block has invalid magic value 0x%08x", + le32_to_cpu(super->hdr.magic)); + ret = -EINVAL; + goto out; + } + + if (calc != super->hdr.crc) { + scoutfs_err(sb, "super block has invalid crc 0x%08x, calculated 0x%08x", + le32_to_cpu(super->hdr.crc), le32_to_cpu(calc)); + ret = -EINVAL; + goto out; + } + + if (le64_to_cpu(super->hdr.blkno) != SCOUTFS_SUPER_BLKNO) { + scoutfs_err(sb, "super block has invalid block number %llu, data read from %llu", + le64_to_cpu(super->hdr.blkno), SCOUTFS_SUPER_BLKNO); + ret = -EINVAL; + goto out; + } + + + if (super->format_hash != cpu_to_le64(SCOUTFS_FORMAT_HASH)) { + scoutfs_err(sb, "super block has invalid format hash 0x%llx, expected 0x%llx", + le64_to_cpu(super->format_hash), + SCOUTFS_FORMAT_HASH); + ret = -EINVAL; + goto out; + } + + /* XXX do we want more rigorous invalid super checking? */ + + if (super->quorum_count == 0 || + super->quorum_count > SCOUTFS_QUORUM_MAX_COUNT) { + scoutfs_err(sb, "super block has invalid quorum count %u, must be > 0 and <= %u", + super->quorum_count, SCOUTFS_QUORUM_MAX_COUNT); + ret = -EINVAL; + goto out; + } + + blkno = (SCOUTFS_QUORUM_BLKNO + SCOUTFS_QUORUM_BLOCKS) >> + SCOUTFS_BLOCK_SM_LG_SHIFT; + if (le64_to_cpu(super->first_meta_blkno) < blkno) { + scoutfs_err(sb, "super block first meta blkno %llu is within quorum blocks", + le64_to_cpu(super->first_meta_blkno)); + ret = -EINVAL; + goto out; + } + + if (le64_to_cpu(super->first_meta_blkno) > + le64_to_cpu(super->last_meta_blkno)) { + scoutfs_err(sb, "super block first meta blkno %llu is greater than last meta blkno %llu", + le64_to_cpu(super->first_meta_blkno), + le64_to_cpu(super->last_meta_blkno)); + ret = -EINVAL; + goto out; + } + + if (le64_to_cpu(super->first_data_blkno) > + le64_to_cpu(super->last_data_blkno)) { + scoutfs_err(sb, "super block first data blkno %llu is greater than last data blkno %llu", + le64_to_cpu(super->first_data_blkno), + le64_to_cpu(super->last_data_blkno)); + ret = -EINVAL; + goto out; + } + + blkno = (i_size_read(sb->s_bdev->bd_inode) >> + SCOUTFS_BLOCK_SM_SHIFT) - 1; + if (le64_to_cpu(super->last_data_blkno) > blkno) { + scoutfs_err(sb, "super block last data blkno %llu is outsite device size last blkno %llu", + le64_to_cpu(super->last_data_blkno), blkno); + ret = -EINVAL; + goto out; + } + +out: + if (ret == 0) + *super_res = *super; + kfree(super); + + return ret; +} + +/* + * Read the super block from meta dev. + */ +int scoutfs_read_super(struct super_block *sb, + struct scoutfs_super_block *super_res) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + + return scoutfs_read_super_from_bdev(sb, sbi->meta_bdev, super_res); +} + +/* + * This needs to be setup after reading the super because it uses the + * fsid found in the super block. + */ +static int scoutfs_debugfs_setup(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + char name[32]; + + snprintf(name, ARRAY_SIZE(name), SCSBF, SCSB_ARGS(sb)); + + sbi->debug_root = debugfs_create_dir(name, scoutfs_debugfs_root); + if (!sbi->debug_root) + return -ENOMEM; + + return 0; +} + +/* + * Calculate a random id for the mount very early, it's used in tracing + * and message output. The system assumes that a rid of 0 can't exist. We're + * also paranoid and avoid rids that are likely the result of bad rng. + */ +static int assign_random_id(struct scoutfs_sb_info *sbi) +{ + unsigned int attempts = 0; + + do { + if (++attempts == 100) + return -EIO; + get_random_bytes(&sbi->rid, sizeof(sbi->rid)); + } while (sbi->rid == 0 || sbi->rid == ~0ULL); + + return 0; +} + +/* + * Ensure superblock copies in metadata and data block devices are valid, and + * fill in in-memory superblock if so. + */ +static int scoutfs_read_supers(struct super_block *sb) +{ + struct scoutfs_super_block *meta_super = NULL; + struct scoutfs_super_block *data_super = NULL; + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + int ret = 0; + + meta_super = kmalloc(sizeof(struct scoutfs_super_block), GFP_NOFS); + data_super = kmalloc(sizeof(struct scoutfs_super_block), GFP_NOFS); + if (!meta_super || !data_super) { + ret = -ENOMEM; + goto out; + } + + ret = scoutfs_read_super_from_bdev(sb, sbi->meta_bdev, meta_super); + if (ret < 0) { + scoutfs_err(sb, "could not get meta_super: error %d", ret); + goto out; + } + + ret = scoutfs_read_super_from_bdev(sb, sb->s_bdev, data_super); + if (ret < 0) { + scoutfs_err(sb, "could not get data_super: error %d", ret); + goto out; + } + + if (!SCOUTFS_IS_META_BDEV(meta_super)) { + scoutfs_err(sb, "meta_super META flag not set"); + ret = -EINVAL; + goto out; + } + + if (SCOUTFS_IS_META_BDEV(data_super)) { + scoutfs_err(sb, "data_super META flag set"); + ret = -EINVAL; + goto out; + } + + if (memcmp(meta_super->uuid, data_super->uuid, SCOUTFS_UUID_BYTES)) { + scoutfs_err(sb, "superblock UUID mismatch"); + ret = -EINVAL; + goto out; + } + + sbi->super = *meta_super; +out: + kfree(meta_super); + kfree(data_super); + return ret; +} + +static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) +{ + struct scoutfs_sb_info *sbi; + struct mount_options opts; + struct block_device *meta_bdev; + struct inode *inode; + int ret; + + trace_scoutfs_fill_super(sb); + + sb->s_magic = SCOUTFS_SUPER_MAGIC; + sb->s_maxbytes = MAX_LFS_FILESIZE; + sb->s_op = &scoutfs_super_ops; + sb->s_export_op = &scoutfs_export_ops; + + /* btree blocks use long lived bh->b_data refs */ + mapping_set_gfp_mask(sb->s_bdev->bd_inode->i_mapping, GFP_NOFS); + + sbi = kzalloc(sizeof(struct scoutfs_sb_info), GFP_KERNEL); + sb->s_fs_info = sbi; + sbi->sb = sb; + if (!sbi) + return -ENOMEM; + + ret = assign_random_id(sbi); + if (ret < 0) + return ret; + + spin_lock_init(&sbi->next_ino_lock); + init_waitqueue_head(&sbi->trans_hold_wq); + spin_lock_init(&sbi->data_wait_root.lock); + sbi->data_wait_root.root = RB_ROOT; + spin_lock_init(&sbi->trans_write_lock); + INIT_DELAYED_WORK(&sbi->trans_write_work, scoutfs_trans_write_func); + init_waitqueue_head(&sbi->trans_write_wq); + scoutfs_sysfs_init_attrs(sb, &sbi->mopts_ssa); + + ret = scoutfs_parse_options(sb, data, &opts); + if (ret) + goto out; + + sbi->opts = opts; + + ret = sb_set_blocksize(sb, SCOUTFS_BLOCK_SM_SIZE); + if (ret != SCOUTFS_BLOCK_SM_SIZE) { + scoutfs_err(sb, "failed to set blocksize, returned %d", ret); + ret = -EIO; + goto out; + } + + meta_bdev = + blkdev_get_by_path(sbi->opts.metadev_path, + SCOUTFS_META_BDEV_MODE, sb); + if (IS_ERR(meta_bdev)) { + scoutfs_err(sb, "could not open metadev: error %ld", + PTR_ERR(meta_bdev)); + ret = PTR_ERR(meta_bdev); + goto out; + } + sbi->meta_bdev = meta_bdev; + ret = set_blocksize(sbi->meta_bdev, SCOUTFS_BLOCK_SM_SIZE); + if (ret != 0) { + scoutfs_err(sb, "failed to set metadev blocksize, returned %d", + ret); + goto out; + } + + ret = scoutfs_read_supers(sb) ?: + scoutfs_debugfs_setup(sb) ?: + scoutfs_setup_sysfs(sb) ?: + scoutfs_setup_counters(sb) ?: + scoutfs_options_setup(sb) ?: + scoutfs_sysfs_create_attrs(sb, &sbi->mopts_ssa, + mount_options_attrs, "mount_options") ?: + scoutfs_setup_triggers(sb) ?: + scoutfs_block_setup(sb) ?: + scoutfs_forest_setup(sb) ?: + scoutfs_item_setup(sb) ?: + scoutfs_inode_setup(sb) ?: + scoutfs_data_setup(sb) ?: + scoutfs_setup_trans(sb) ?: + scoutfs_lock_setup(sb) ?: + scoutfs_net_setup(sb) ?: + scoutfs_quorum_setup(sb) ?: + scoutfs_server_setup(sb) ?: + scoutfs_client_setup(sb) ?: + scoutfs_lock_rid(sb, SCOUTFS_LOCK_WRITE, 0, sbi->rid, + &sbi->rid_lock) ?: + scoutfs_trans_get_log_trees(sb) ?: + scoutfs_srch_setup(sb); + if (ret) + goto out; + + inode = scoutfs_iget(sb, SCOUTFS_ROOT_INO); + if (IS_ERR(inode)) { + ret = PTR_ERR(inode); + goto out; + } + + sb->s_root = d_make_root(inode); + if (!sb->s_root) { + ret = -ENOMEM; + goto out; + } + + ret = scoutfs_client_advance_seq(sb, &sbi->trans_seq); + if (ret) + goto out; + + scoutfs_trans_restart_sync_deadline(sb); +// scoutfs_scan_orphans(sb); + ret = 0; +out: + /* on error, generic_shutdown_super calls put_super if s_root */ + if (ret && !sb->s_root) + scoutfs_put_super(sb); + + return ret; +} + +static struct dentry *scoutfs_mount(struct file_system_type *fs_type, int flags, + const char *dev_name, void *data) +{ + return mount_bdev(fs_type, flags, dev_name, data, scoutfs_fill_super); +} + +/* + * kill_block_super eventually calls ->put_super if s_root is set + */ +static void scoutfs_kill_sb(struct super_block *sb) +{ + trace_scoutfs_kill_sb(sb); + + kill_block_super(sb); +} + +static struct file_system_type scoutfs_fs_type = { + .owner = THIS_MODULE, + .name = "scoutfs", + .mount = scoutfs_mount, + .kill_sb = scoutfs_kill_sb, + .fs_flags = FS_REQUIRES_DEV, +}; +MODULE_ALIAS_FS("scoutfs"); + +/* safe to call at any failure point in _init */ +static void teardown_module(void) +{ + debugfs_remove(scoutfs_debugfs_root); + scoutfs_dir_exit(); + scoutfs_inode_exit(); + scoutfs_sysfs_exit(); +} + +static int __init scoutfs_module_init(void) +{ + int ret; + + /* + * gcc only recently learned to let __attribute__(section) add + * SHT_NOTE notes. But the assembler always could. + */ + __asm__ __volatile__ ( + ".section .note.git_describe,\"a\"\n" + ".string \""SCOUTFS_GIT_DESCRIBE"\\n\"\n" + ".previous\n"); + + scoutfs_init_counters(); + + ret = scoutfs_sysfs_init(); + if (ret) + return ret; + + scoutfs_debugfs_root = debugfs_create_dir("scoutfs", NULL); + if (!scoutfs_debugfs_root) { + ret = -ENOMEM; + goto out; + } + ret = scoutfs_inode_init() ?: + scoutfs_dir_init() ?: + register_filesystem(&scoutfs_fs_type); +out: + if (ret) + teardown_module(); + return ret; +} +module_init(scoutfs_module_init) + +static void __exit scoutfs_module_exit(void) +{ + unregister_filesystem(&scoutfs_fs_type); + teardown_module(); +} +module_exit(scoutfs_module_exit) + +MODULE_AUTHOR("Zach Brown "); +MODULE_LICENSE("GPL"); +MODULE_INFO(git_describe, SCOUTFS_GIT_DESCRIBE); diff --git a/kmod/src/super.h b/kmod/src/super.h new file mode 100644 index 00000000..13912bdc --- /dev/null +++ b/kmod/src/super.h @@ -0,0 +1,145 @@ +#ifndef _SCOUTFS_SUPER_H_ +#define _SCOUTFS_SUPER_H_ + +#include +#include + +#include "format.h" +#include "options.h" +#include "data.h" +#include "sysfs.h" + +struct scoutfs_counters; +struct scoutfs_triggers; +struct manifest; +struct data_info; +struct trans_info; +struct lock_info; +struct lock_server_info; +struct client_info; +struct server_info; +struct inode_sb_info; +struct btree_info; +struct sysfs_info; +struct options_sb_info; +struct net_info; +struct block_info; +struct forest_info; +struct srch_info; + +struct scoutfs_sb_info { + struct super_block *sb; + + /* assigned once at the start of each mount, read-only */ + u64 rid; + struct scoutfs_lock *rid_lock; + + struct scoutfs_super_block super; + + struct block_device *meta_bdev; + + spinlock_t next_ino_lock; + + struct data_info *data_info; + struct inode_sb_info *inode_sb_info; + struct btree_info *btree_info; + struct net_info *net_info; + struct quorum_info *quorum_info; + struct block_info *block_info; + struct forest_info *forest_info; + struct srch_info *srch_info; + struct item_cache_info *item_cache_info; + + wait_queue_head_t trans_hold_wq; + struct task_struct *trans_task; + + /* tracks tasks waiting for data extents */ + struct scoutfs_data_wait_root data_wait_root; + + spinlock_t trans_write_lock; + u64 trans_write_count; + u64 trans_seq; + int trans_write_ret; + struct delayed_work trans_write_work; + wait_queue_head_t trans_write_wq; + struct workqueue_struct *trans_write_workq; + bool trans_deadline_expired; + + struct trans_info *trans_info; + struct lock_info *lock_info; + struct lock_server_info *lock_server_info; + struct client_info *client_info; + struct server_info *server_info; + struct sysfs_info *sfsinfo; + + struct scoutfs_counters *counters; + struct scoutfs_triggers *triggers; + + struct mount_options opts; + struct options_sb_info *options; + struct scoutfs_sysfs_attrs mopts_ssa; + + struct dentry *debug_root; + + bool shutdown; + + unsigned long corruption_messages_once[SC_NR_LONGS]; +}; + +static inline struct scoutfs_sb_info *SCOUTFS_SB(struct super_block *sb) +{ + return sb->s_fs_info; +} + +static inline bool SCOUTFS_HAS_SBI(struct super_block *sb) +{ + return (sb != NULL) && (SCOUTFS_SB(sb) != NULL); +} + +static inline bool SCOUTFS_IS_META_BDEV(struct scoutfs_super_block *super_block) +{ + return !!(le64_to_cpu(super_block->flags) & SCOUTFS_FLAG_IS_META_BDEV); +} + +#define SCOUTFS_META_BDEV_MODE (FMODE_READ | FMODE_WRITE | FMODE_EXCL) + +/* + * A small string embedded in messages that's used to identify a + * specific mount. It's the three most significant bytes of the fsid + * and the rid. That gives us a strong chance of avoiding collisions + * with typical numbers of mounts. We give it a bit of structure to + * make it searchable and to be able to identify format changes, should + * we need to. The fsid will be 0 until the super has been read and the + * fsid discovered. + */ +#define SCSBF "f.%.06x.r.%.06x" +#define SCSB_SHIFT (64 - (8 * 3)) +#define SCSB_LEFR_ARGS(fsid, rid) \ + (int)(le64_to_cpu(fsid) >> SCSB_SHIFT), \ + (int)(le64_to_cpu(rid) >> SCSB_SHIFT) +#define SCSB_ARGS(sb) \ + (int)(le64_to_cpu(SCOUTFS_SB(sb)->super.hdr.fsid) >> SCSB_SHIFT), \ + (int)(SCOUTFS_SB(sb)->rid >> SCSB_SHIFT) +#define SCSB_TRACE_FIELDS \ + __field(__u64, fsid) \ + __field(__u64, rid) +#define SCSB_TRACE_ASSIGN(sb) \ + __entry->fsid = SCOUTFS_HAS_SBI(sb) ? \ + le64_to_cpu(SCOUTFS_SB(sb)->super.hdr.fsid) : 0;\ + __entry->rid = SCOUTFS_HAS_SBI(sb) ? \ + SCOUTFS_SB(sb)->rid : 0; +#define SCSB_TRACE_ARGS \ + (int)(__entry->fsid >> SCSB_SHIFT), \ + (int)(__entry->rid >> SCSB_SHIFT) + +int scoutfs_read_super(struct super_block *sb, + struct scoutfs_super_block *super_res); +int scoutfs_write_super(struct super_block *sb, + struct scoutfs_super_block *super); + +/* to keep this out of the ioctl.h public interface definition */ +long scoutfs_ioctl(struct file *file, unsigned int cmd, unsigned long arg); + +__le64 scoutfs_clock_sync_id(void); + +#endif diff --git a/kmod/src/sysfs.c b/kmod/src/sysfs.c new file mode 100644 index 00000000..bea667d0 --- /dev/null +++ b/kmod/src/sysfs.c @@ -0,0 +1,253 @@ +/* + * Copyright (C) 2017 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include + +#include "super.h" +#include "sysfs.h" + +static struct kset *scoutfs_kset; + +struct sysfs_info { + struct super_block *sb; + struct kobject sb_id_kobj; + struct completion sb_id_comp; +}; + +#define KOBJ_TO_SB(kobj, which) \ + container_of(kobj, struct sysfs_info, which)->sb + +struct attr_funcs { + struct attribute attr; + ssize_t (*show)(struct kobject *kobj, struct attribute *attr, + char *buf); +}; + +#define ATTR_FUNCS_RO(_name) \ + static struct attr_funcs _name##_attr_funcs = __ATTR_RO(_name) + +static ssize_t fsid_show(struct kobject *kobj, struct attribute *attr, + char *buf) +{ + struct super_block *sb = KOBJ_TO_SB(kobj, sb_id_kobj); + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + + return snprintf(buf, PAGE_SIZE, "%016llx\n", + le64_to_cpu(super->hdr.fsid)); +} +ATTR_FUNCS_RO(fsid); + +static ssize_t rid_show(struct kobject *kobj, struct attribute *attr, char *buf) +{ + struct super_block *sb = KOBJ_TO_SB(kobj, sb_id_kobj); + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + + return snprintf(buf, PAGE_SIZE, "%016llx\n", sbi->rid); +} +ATTR_FUNCS_RO(rid); + +/* + * ops are defined per type, not per attribute. To have attributes with + * different types that want different funcs we wrap them with a struct + * that has per-type funcs. + */ +static ssize_t attr_funcs_show(struct kobject *kobj, struct attribute *attr, + char *buf) +{ + struct attr_funcs *af = container_of(attr, struct attr_funcs, attr); + + return af->show(kobj, attr, buf); +} + +#define KTYPE(_name) \ + static void _name##_release(struct kobject *kobj) \ + { \ + struct sysfs_info *sfsinfo; \ + \ + sfsinfo = container_of(kobj, struct sysfs_info, _name##_kobj);\ + \ + complete(&sfsinfo->_name##_comp); \ + } \ + static const struct sysfs_ops _name##_sysfs_ops = { \ + .show = attr_funcs_show, \ + }; \ + \ + static struct kobj_type _name##_ktype = { \ + .default_attrs = _name##_attrs, \ + .sysfs_ops = &_name##_sysfs_ops, \ + .release = _name##_release, \ + }; + + +static struct attribute *sb_id_attrs[] = { + &fsid_attr_funcs.attr, + &rid_attr_funcs.attr, + NULL, +}; +KTYPE(sb_id); + +struct kobject *scoutfs_sysfs_sb_dir(struct super_block *sb) +{ + struct sysfs_info *sfsinfo = SCOUTFS_SB(sb)->sfsinfo; + + return &sfsinfo->sb_id_kobj; +} + +static void kobj_del_put_wait(struct kobject *kobj, struct completion *comp) +{ + kobject_del(kobj); + kobject_put(kobj); + wait_for_completion(comp); +} + +#define shutdown_kobj(sfinfo, _name) \ + kobj_del_put_wait(&sfsinfo->_name##_kobj, &sfsinfo->_name##_comp) + +static void scoutfs_sysfs_release(struct kobject *kobj) +{ + DECLARE_SCOUTFS_SYSFS_ATTRS(ssa, kobj); + + complete(&ssa->comp); +} + +void scoutfs_sysfs_init_attrs(struct super_block *sb, + struct scoutfs_sysfs_attrs *ssa) +{ + ssa->name = NULL; +} + +/* + * If this returns success then the file will be visible and show can + * be called until unmount. + */ +int scoutfs_sysfs_create_attrs(struct super_block *sb, + struct scoutfs_sysfs_attrs *ssa, + struct attribute **attrs, char *fmt, ...) +{ + va_list args; + size_t name_len; + size_t size; + int ret; + + /* ssa should have seen init or destroy */ + if (WARN_ON_ONCE(ssa->name != NULL)) + return -EINVAL; + + ssa->sb = sb; + init_completion(&ssa->comp); + ssa->ktype.default_attrs = attrs; + ssa->ktype.sysfs_ops = &kobj_sysfs_ops; + ssa->ktype.release = scoutfs_sysfs_release; + + va_start(args, fmt); + name_len = vsnprintf(NULL, 0, fmt, args); + va_end(args); + if (WARN_ON_ONCE(name_len < 1 || name_len > NAME_MAX)) { + ret = -EINVAL; + goto out; + } + + size = name_len + 1; /* with null */ + + ssa->name = kmalloc(size, GFP_KERNEL); + if (!ssa->name) { + ret = -ENOMEM; + goto out; + } + + va_start(args, fmt); + ret = vsnprintf(ssa->name, size, fmt, args); + va_end(args); + if (ret != name_len) { + ret = -EINVAL; + goto out; + } + + ret = kobject_init_and_add(&ssa->kobj, &ssa->ktype, + scoutfs_sysfs_sb_dir(sb), "%s", ssa->name); +out: + if (ret) { + kfree(ssa->name); + ssa->name = NULL; + } + + return ret; +} + +void scoutfs_sysfs_destroy_attrs(struct super_block *sb, + struct scoutfs_sysfs_attrs *ssa) +{ + if (ssa->name) { + kobject_del(&ssa->kobj); + kobject_put(&ssa->kobj); + wait_for_completion(&ssa->comp); + kfree(ssa->name); + ssa->name = NULL; + } +} + +/* + * Only the return from kobj_init_and_add() tells us if the kobj needs + * to be cleaned up or not. This must manually clean up the kobjs and + * only leave full cleanup to _destroy_. + */ +int scoutfs_setup_sysfs(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct sysfs_info *sfsinfo; + int ret; + + sfsinfo = kzalloc(sizeof(struct sysfs_info), GFP_KERNEL); + if (!sfsinfo) + return -ENOMEM; + + sfsinfo->sb = sb; + sbi->sfsinfo = sfsinfo; + + init_completion(&sfsinfo->sb_id_comp); + ret = kobject_init_and_add(&sfsinfo->sb_id_kobj, &sb_id_ktype, + &scoutfs_kset->kobj, SCSBF, SCSB_ARGS(sb)); + if (ret) + kfree(sfsinfo); + + return ret; +} + +void scoutfs_destroy_sysfs(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct sysfs_info *sfsinfo = sbi->sfsinfo; + + if (sfsinfo) { + shutdown_kobj(sfsinfo, sb_id); + + kfree(sfsinfo); + sbi->sfsinfo = NULL; + } +} + +int __init scoutfs_sysfs_init(void) +{ + scoutfs_kset = kset_create_and_add("scoutfs", NULL, fs_kobj); + if (!scoutfs_kset) + return -ENOMEM; + + return 0; +} + +void __exit scoutfs_sysfs_exit(void) +{ + if (scoutfs_kset) + kset_unregister(scoutfs_kset); +} diff --git a/kmod/src/sysfs.h b/kmod/src/sysfs.h new file mode 100644 index 00000000..73788c00 --- /dev/null +++ b/kmod/src/sysfs.h @@ -0,0 +1,51 @@ +#ifndef _SCOUTFS_SYSFS_H_ +#define _SCOUTFS_SYSFS_H_ + +#include + +/* + * We have some light wrappers around sysfs attributes to make it safe + * to tear down the attributes before freeing the data they describe. + */ + +#define SCOUTFS_ATTR_RO(_name) \ + static struct kobj_attribute scoutfs_attr_##_name = __ATTR_RO(_name) + +#define SCOUTFS_ATTR_PTR(_name) \ + &scoutfs_attr_##_name.attr + +struct scoutfs_sysfs_attrs { + struct super_block *sb; + char *name; + struct completion comp; + + struct kobject kobj; + struct kobj_type ktype; +}; + +#define SCOUTFS_SYSFS_ATTRS(kobj) \ + container_of(kobj, struct scoutfs_sysfs_attrs, kobj) + +#define SCOUTFS_SYSFS_ATTRS_SB(kobj) \ + (SCOUTFS_SYSFS_ATTRS(kobj)->sb) + +#define DECLARE_SCOUTFS_SYSFS_ATTRS(name, kobj) \ + struct scoutfs_sysfs_attrs *ssa = SCOUTFS_SYSFS_ATTRS(kobj) + +void scoutfs_sysfs_init_attrs(struct super_block *sb, + struct scoutfs_sysfs_attrs *ssa); +int scoutfs_sysfs_create_attrs(struct super_block *sb, + struct scoutfs_sysfs_attrs *ssa, + struct attribute **attrs, char *fmt, ...); +void scoutfs_sysfs_destroy_attrs(struct super_block *sb, + struct scoutfs_sysfs_attrs *ssa); + +struct kobject *scoutfs_sysfs_sb_dir(struct super_block *sb); + +int scoutfs_setup_sysfs(struct super_block *sb); +void scoutfs_destroy_sysfs(struct super_block *sb); + +int __init scoutfs_sysfs_init(void); +void __exit scoutfs_sysfs_exit(void); + +#endif diff --git a/kmod/src/trans.c b/kmod/src/trans.c new file mode 100644 index 00000000..e024c244 --- /dev/null +++ b/kmod/src/trans.c @@ -0,0 +1,614 @@ +/* + * Copyright (C) 2016 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include +#include +#include +#include +#include + +#include "super.h" +#include "trans.h" +#include "data.h" +#include "forest.h" +#include "counters.h" +#include "client.h" +#include "inode.h" +#include "alloc.h" +#include "block.h" +#include "msg.h" +#include "item.h" +#include "scoutfs_trace.h" + +/* + * scoutfs blocks are written in atomic transactions. + * + * Writers hold transactions to dirty blocks. The transaction can't be + * written until these active writers release the transaction. We don't + * track the relationships between dirty blocks so there's only ever one + * transaction being built. + * + * The copy of the on-disk super block in the fs sb info has its header + * sequence advanced so that new dirty blocks inherit this dirty + * sequence number. It's only advanced once all those dirty blocks are + * reachable after having first written them all out and then the new + * super with that seq. It's first incremented at mount. + * + * Unfortunately writers can nest. We don't bother trying to special + * case holding a transaction that you're already holding because that + * requires per-task storage. We just let anyone hold transactions + * regardless of waiters waiting to write, which risks waiters waiting a + * very long time. + */ + +/* sync dirty data at least this often */ +#define TRANS_SYNC_DELAY (HZ * 10) + +/* + * XXX move the rest of the super trans_ fields here. + */ +struct trans_info { + spinlock_t lock; + unsigned reserved_items; + unsigned reserved_vals; + unsigned holders; + bool writing; + + struct scoutfs_log_trees lt; + struct scoutfs_alloc alloc; + struct scoutfs_block_writer wri; +}; + +#define DECLARE_TRANS_INFO(sb, name) \ + struct trans_info *name = SCOUTFS_SB(sb)->trans_info + +static bool drained_holders(struct trans_info *tri) +{ + bool drained; + + spin_lock(&tri->lock); + tri->writing = true; + drained = tri->holders == 0; + spin_unlock(&tri->lock); + + return drained; +} + +static int commit_btrees(struct super_block *sb) +{ + DECLARE_TRANS_INFO(sb, tri); + struct scoutfs_log_trees lt; + + lt = tri->lt; + lt.meta_avail = tri->alloc.avail; + lt.meta_freed = tri->alloc.freed; + scoutfs_forest_get_btrees(sb, <); + scoutfs_data_get_btrees(sb, <); + + return scoutfs_client_commit_log_trees(sb, <); +} + +/* + * This gets all the resources from the server that the client will + * need during the transaction. + */ +int scoutfs_trans_get_log_trees(struct super_block *sb) +{ + DECLARE_TRANS_INFO(sb, tri); + struct scoutfs_log_trees lt; + int ret = 0; + + ret = scoutfs_client_get_log_trees(sb, <); + if (ret == 0) { + tri->lt = lt; + scoutfs_alloc_init(&tri->alloc, <.meta_avail, <.meta_freed); + scoutfs_block_writer_init(sb, &tri->wri); + + scoutfs_forest_init_btrees(sb, &tri->alloc, &tri->wri, <); + scoutfs_data_init_btrees(sb, &tri->alloc, &tri->wri, <); + } + return ret; +} + +bool scoutfs_trans_has_dirty(struct super_block *sb) +{ + DECLARE_TRANS_INFO(sb, tri); + + return scoutfs_block_writer_has_dirty(sb, &tri->wri); +} + +/* + * This work func is responsible for writing out all the dirty blocks + * that make up the current dirty transaction. It prevents writers from + * holding a transaction so it doesn't have to worry about blocks being + * dirtied while it is working. + * + * In the course of doing its work this task might need to use write + * functions that would try to hold the transaction. We record the task + * whose committing the transaction so that holding won't deadlock. + * + * Any dirty block had to have allocated a new blkno which would have + * created dirty allocator metadata blocks. We can avoid writing + * entirely if we don't have any dirty metadata blocks. This is + * important because we don't try to serialize this work during + * unmount.. we can execute as the vfs is shutting down.. we need to + * decide that nothing is dirty without calling the vfs at all. + * + * We first try to sync the dirty inodes and write their dirty data blocks, + * then we write all our dirty metadata blocks, and only when those succeed + * do we write the new super that references all of these newly written blocks. + * + * If there are write errors then blocks are kept dirty in memory and will + * be written again at the next sync. + */ +void scoutfs_trans_write_func(struct work_struct *work) +{ + struct scoutfs_sb_info *sbi = container_of(work, struct scoutfs_sb_info, + trans_write_work.work); + struct super_block *sb = sbi->sb; + DECLARE_TRANS_INFO(sb, tri); + u64 trans_seq = sbi->trans_seq; + char *s = NULL; + int ret = 0; + + sbi->trans_task = current; + + wait_event(sbi->trans_hold_wq, drained_holders(tri)); + + trace_scoutfs_trans_write_func(sb, + scoutfs_block_writer_dirty_bytes(sb, &tri->wri)); + + if (!scoutfs_block_writer_has_dirty(sb, &tri->wri) && + !scoutfs_item_dirty_pages(sb)) { + if (sbi->trans_deadline_expired) { + /* + * If we're not writing data then we only advance the + * seq at the sync deadline interval. This keeps idle + * mounts from pinning a seq and stopping readers of the + * seq indices but doesn't send a message for every sync + * syscall. + */ + ret = scoutfs_client_advance_seq(sb, &trans_seq); + if (ret < 0) + s = "clean advance seq"; + } + goto out; + } + + if (sbi->trans_deadline_expired) + scoutfs_inc_counter(sb, trans_commit_timer); + + scoutfs_inc_counter(sb, trans_commit_written); + + /* XXX this all needs serious work for dealing with errors */ + ret = (s = "data submit", scoutfs_inode_walk_writeback(sb, true)) ?: + (s = "item dirty", scoutfs_item_write_dirty(sb)) ?: + (s = "data prepare", scoutfs_data_prepare_commit(sb)) ?: + (s = "alloc prepare", scoutfs_alloc_prepare_commit(sb, + &tri->alloc, &tri->wri)) ?: + (s = "meta write", scoutfs_block_writer_write(sb, &tri->wri)) ?: + (s = "data wait", scoutfs_inode_walk_writeback(sb, false)) ?: + (s = "commit log trees", commit_btrees(sb)) ?: + scoutfs_item_write_done(sb) ?: + (s = "advance seq", scoutfs_client_advance_seq(sb, &trans_seq)) ?: + (s = "get log trees", scoutfs_trans_get_log_trees(sb)); +out: + if (ret < 0) + scoutfs_err(sb, "critical transaction commit failure: %s, %d", + s, ret); + + spin_lock(&sbi->trans_write_lock); + sbi->trans_write_count++; + sbi->trans_write_ret = ret; + sbi->trans_seq = trans_seq; + spin_unlock(&sbi->trans_write_lock); + wake_up(&sbi->trans_write_wq); + + spin_lock(&tri->lock); + tri->writing = false; + spin_unlock(&tri->lock); + + wake_up(&sbi->trans_hold_wq); + + sbi->trans_task = NULL; + + scoutfs_trans_restart_sync_deadline(sb); +} + +struct write_attempt { + u64 count; + int ret; +}; + +/* this is called as a wait_event() condition so it can't change task state */ +static int write_attempted(struct scoutfs_sb_info *sbi, + struct write_attempt *attempt) +{ + int done = 1; + + spin_lock(&sbi->trans_write_lock); + if (sbi->trans_write_count > attempt->count) + attempt->ret = sbi->trans_write_ret; + else + done = 0; + spin_unlock(&sbi->trans_write_lock); + + return done; +} + + +/* + * We always have delayed sync work pending but the caller wants it + * to execute immediately. + */ +static void queue_trans_work(struct scoutfs_sb_info *sbi) +{ + sbi->trans_deadline_expired = false; + mod_delayed_work(sbi->trans_write_workq, &sbi->trans_write_work, 0); +} + +/* + * Wait for a trans commit to finish and return its error code. There + * can already be one in flight that we end up waiting for the + * completion of. This is safe because dirtying and trans commits are + * serialized. There's no way that there could have been dirty data + * before the caller got here that wouldn't be covered by a commit + * that's in flight. + */ +int scoutfs_trans_sync(struct super_block *sb, int wait) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct write_attempt attempt; + int ret; + + + if (!wait) { + queue_trans_work(sbi); + return 0; + } + + spin_lock(&sbi->trans_write_lock); + attempt.count = sbi->trans_write_count; + spin_unlock(&sbi->trans_write_lock); + + queue_trans_work(sbi); + + ret = wait_event_interruptible(sbi->trans_write_wq, + write_attempted(sbi, &attempt)); + if (ret == 0) + ret = attempt.ret; + + return ret; +} + +int scoutfs_file_fsync(struct file *file, loff_t start, loff_t end, + int datasync) +{ + struct super_block *sb = file_inode(file)->i_sb; + + scoutfs_inc_counter(sb, trans_commit_fsync); + return scoutfs_trans_sync(sb, 1); +} + +void scoutfs_trans_restart_sync_deadline(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + + sbi->trans_deadline_expired = true; + mod_delayed_work(sbi->trans_write_workq, &sbi->trans_write_work, + TRANS_SYNC_DELAY); +} + +/* + * Each thread reserves space in the segment for their dirty items while + * they hold the transaction. This is calculated before the first + * transaction hold is acquired. It includes all the potential nested + * item manipulation that could happen with the transaction held. + * Including nested holds avoids having to deal with writing out partial + * transactions while a caller still holds the transaction. + */ +#define SCOUTFS_RESERVATION_MAGIC 0xd57cd13b +struct scoutfs_reservation { + unsigned magic; + unsigned holders; + struct scoutfs_item_count reserved; + struct scoutfs_item_count actual; +}; + +/* + * Try to hold the transaction. If a caller already holds the trans then + * we piggy back on their hold. We wait if the writer is trying to + * write out the transation. And if our items won't fit then we kick off + * a write. + * + * This is called as a condition for wait_event. It is very limited in + * the locking (blocking) it can do because the caller has set the task + * state before testing the condition safely race with waking after + * setting the condition. Our checking the amount of dirty metadata + * blocks and free data blocks is racy, but we don't mind the risk of + * delaying or prematurely forcing commits. + */ +static bool acquired_hold(struct super_block *sb, + struct scoutfs_reservation *rsv, + const struct scoutfs_item_count *cnt) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + DECLARE_TRANS_INFO(sb, tri); + bool acquired = false; + unsigned items; + unsigned vals; + + spin_lock(&tri->lock); + + trace_scoutfs_trans_acquired_hold(sb, cnt, rsv, rsv->holders, + &rsv->reserved, &rsv->actual, + tri->holders, tri->writing, + tri->reserved_items, + tri->reserved_vals); + + /* use a caller's existing reservation */ + if (rsv->holders) + goto hold; + + /* wait until the writing thread is finished */ + if (tri->writing) + goto out; + + /* see if we can reserve space for our item count */ + items = tri->reserved_items + cnt->items; + vals = tri->reserved_vals + cnt->vals; + + /* + * In theory each dirty item page could be straddling two full + * blocks, requiring 4 allocations for each item cache page. + * That's much too conservative, typically many dirty item cache + * pages that are near each other all land in one block. This + * rough estimate is still so far beyond what typically happens + * that it accounts for having to dirty parent blocks and + * whatever dirtying is done during the transaction hold. + */ + if (scoutfs_alloc_meta_low(sb, &tri->alloc, + scoutfs_item_dirty_pages(sb) * 2)) { + scoutfs_inc_counter(sb, trans_commit_dirty_meta_full); + queue_trans_work(sbi); + goto out; + } + + /* + * Extent modifications can use meta allocators without creating + * dirty items so we have to check the meta alloc specifically. + * The size of the client's avail and freed roots are bound so + * we're unlikely to need very many block allocations per + * transaction hold. XXX This should be more precisely tuned. + */ + if (scoutfs_alloc_meta_low(sb, &tri->alloc, 16)) { + scoutfs_inc_counter(sb, trans_commit_meta_alloc_low); + queue_trans_work(sbi); + goto out; + } + + /* Try to refill data allocator before premature enospc */ + if (scoutfs_data_alloc_free_bytes(sb) <= SCOUTFS_TRANS_DATA_ALLOC_LWM) { + scoutfs_inc_counter(sb, trans_commit_data_alloc_low); + queue_trans_work(sbi); + goto out; + } + + tri->reserved_items = items; + tri->reserved_vals = vals; + + rsv->reserved.items = cnt->items; + rsv->reserved.vals = cnt->vals; + +hold: + rsv->holders++; + tri->holders++; + acquired = true; + +out: + + spin_unlock(&tri->lock); + + return acquired; +} + +int scoutfs_hold_trans(struct super_block *sb, + const struct scoutfs_item_count cnt) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_reservation *rsv; + int ret; + + /* + * Caller shouldn't provide garbage counts, nor counts that + * can't fit in segments by themselves. + */ + if (WARN_ON_ONCE(cnt.items <= 0 || cnt.vals < 0)) + return -EINVAL; + + if (current == sbi->trans_task) + return 0; + + rsv = current->journal_info; + if (rsv == NULL) { + rsv = kzalloc(sizeof(struct scoutfs_reservation), GFP_NOFS); + if (!rsv) + return -ENOMEM; + + rsv->magic = SCOUTFS_RESERVATION_MAGIC; + current->journal_info = rsv; + } + + BUG_ON(rsv->magic != SCOUTFS_RESERVATION_MAGIC); + + ret = wait_event_interruptible(sbi->trans_hold_wq, + acquired_hold(sb, rsv, &cnt)); + if (ret && rsv->holders == 0) { + current->journal_info = NULL; + kfree(rsv); + } + return ret; +} + +/* + * Return true if the current task has a transaction held. That is, + * true if the current transaction can't finish and be written out if + * the current task blocks. + */ +bool scoutfs_trans_held(void) +{ + struct scoutfs_reservation *rsv = current->journal_info; + + return rsv && rsv->magic == SCOUTFS_RESERVATION_MAGIC; +} + +/* + * Record a transaction holder's individual contribution to the dirty + * items in the current transaction. We're making sure that the + * reservation matches the possible item manipulations while they hold + * the reservation. + * + * It is possible and legitimate for an individual contribution to be + * negative if they delete dirty items. The item cache makes sure that + * the total dirty item count doesn't fall below zero. + */ +void scoutfs_trans_track_item(struct super_block *sb, signed items, + signed vals) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_reservation *rsv = current->journal_info; + + if (current == sbi->trans_task) + return; + + BUG_ON(!rsv || rsv->magic != SCOUTFS_RESERVATION_MAGIC); + + rsv->actual.items += items; + rsv->actual.vals += vals; + + trace_scoutfs_trans_track_item(sb, items, vals, rsv->actual.items, + rsv->actual.vals, rsv->reserved.items, + rsv->reserved.vals); + + WARN_ON_ONCE(rsv->actual.items > rsv->reserved.items); + WARN_ON_ONCE(rsv->actual.vals > rsv->reserved.vals); +} + +/* + * As we drop the last hold in the reservation we try and wake other + * hold attempts that were waiting for space. As we drop the last trans + * holder we try to wake a writing thread that was waiting for us to + * finish. + */ +void scoutfs_release_trans(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_reservation *rsv; + DECLARE_TRANS_INFO(sb, tri); + bool wake = false; + + if (current == sbi->trans_task) + return; + + rsv = current->journal_info; + BUG_ON(!rsv || rsv->magic != SCOUTFS_RESERVATION_MAGIC); + + spin_lock(&tri->lock); + + trace_scoutfs_release_trans(sb, rsv, rsv->holders, &rsv->reserved, + &rsv->actual, tri->holders, tri->writing, + tri->reserved_items, tri->reserved_vals); + + BUG_ON(rsv->holders <= 0); + BUG_ON(tri->holders <= 0); + + if (--rsv->holders == 0) { + tri->reserved_items -= rsv->reserved.items; + tri->reserved_vals -= rsv->reserved.vals; + current->journal_info = NULL; + kfree(rsv); + wake = true; + } + + if (--tri->holders == 0) + wake = true; + + spin_unlock(&tri->lock); + + if (wake) + wake_up(&sbi->trans_hold_wq); +} + +/* + * Return the current transaction sequence. Whether this is racing with + * the transaction write thread is entirely dependent on the caller's + * context. + */ +u64 scoutfs_trans_sample_seq(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + u64 ret; + + spin_lock(&sbi->trans_write_lock); + ret = sbi->trans_seq; + spin_unlock(&sbi->trans_write_lock); + + return ret; +} + +int scoutfs_setup_trans(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct trans_info *tri; + + tri = kzalloc(sizeof(struct trans_info), GFP_KERNEL); + if (!tri) + return -ENOMEM; + + spin_lock_init(&tri->lock); + scoutfs_block_writer_init(sb, &tri->wri); + + sbi->trans_write_workq = alloc_workqueue("scoutfs_trans", + WQ_UNBOUND, 1); + if (!sbi->trans_write_workq) { + kfree(tri); + return -ENOMEM; + } + + sbi->trans_info = tri; + + return 0; +} + +/* + * kill_sb calls sync before getting here so we know that dirty data + * should be in flight. We just have to wait for it to quiesce. + */ +void scoutfs_shutdown_trans(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + DECLARE_TRANS_INFO(sb, tri); + + if (tri) { + scoutfs_block_writer_forget_all(sb, &tri->wri); + if (sbi->trans_write_workq) { + cancel_delayed_work_sync(&sbi->trans_write_work); + destroy_workqueue(sbi->trans_write_workq); + /* trans work schedules after shutdown see null */ + sbi->trans_write_workq = NULL; + } + kfree(tri); + sbi->trans_info = NULL; + } +} diff --git a/kmod/src/trans.h b/kmod/src/trans.h new file mode 100644 index 00000000..f1b50f8f --- /dev/null +++ b/kmod/src/trans.h @@ -0,0 +1,31 @@ +#ifndef _SCOUTFS_TRANS_H_ +#define _SCOUTFS_TRANS_H_ + +/* the server will attempt to fill data allocs for each trans */ +#define SCOUTFS_TRANS_DATA_ALLOC_HWM (2ULL * 1024 * 1024 * 1024) +/* the client will force commits if data allocators get too low */ +#define SCOUTFS_TRANS_DATA_ALLOC_LWM (256ULL * 1024 * 1024) + +#include "count.h" + +void scoutfs_trans_write_func(struct work_struct *work); +int scoutfs_trans_sync(struct super_block *sb, int wait); +int scoutfs_file_fsync(struct file *file, loff_t start, loff_t end, + int datasync); +void scoutfs_trans_restart_sync_deadline(struct super_block *sb); + +int scoutfs_hold_trans(struct super_block *sb, + const struct scoutfs_item_count cnt); +bool scoutfs_trans_held(void); +void scoutfs_release_trans(struct super_block *sb); +u64 scoutfs_trans_sample_seq(struct super_block *sb); +void scoutfs_trans_track_item(struct super_block *sb, signed items, + signed vals); + +int scoutfs_trans_get_log_trees(struct super_block *sb); +bool scoutfs_trans_has_dirty(struct super_block *sb); + +int scoutfs_setup_trans(struct super_block *sb); +void scoutfs_shutdown_trans(struct super_block *sb); + +#endif diff --git a/kmod/src/triggers.c b/kmod/src/triggers.c new file mode 100644 index 00000000..a94f2b65 --- /dev/null +++ b/kmod/src/triggers.c @@ -0,0 +1,122 @@ +/* + * Copyright (C) 2017 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include +#include +#include + +#include "super.h" +#include "triggers.h" + +/* + * We have debugfs files we can write to which arm triggers which + * atomically fire once for testing or debugging. + */ + +/* + * The atomic cachelines are kept hot and shared by being read by fast + * paths. They're very rarely modified by debugfs writes which arm them + * and then the next read will atomically clear and return true. + */ +struct scoutfs_triggers { + struct dentry *dir; + atomic_t atomics[SCOUTFS_TRIGGER_NR]; +}; + +#define DECLARE_TRIGGERS(sb, name) \ + struct scoutfs_triggers *name = SCOUTFS_SB(sb)->triggers + +static char *names[] = { + [SCOUTFS_TRIGGER_BTREE_STALE_READ] = "btree_stale_read", + [SCOUTFS_TRIGGER_BTREE_ADVANCE_RING_HALF] = "btree_advance_ring_half", + [SCOUTFS_TRIGGER_HARD_STALE_ERROR] = "hard_stale_error", + [SCOUTFS_TRIGGER_SEG_STALE_READ] = "seg_stale_read", + [SCOUTFS_TRIGGER_STATFS_LOCK_PURGE] = "statfs_lock_purge", +}; + +bool scoutfs_trigger_test_and_clear(struct super_block *sb, unsigned int t) +{ + DECLARE_TRIGGERS(sb, triggers); + atomic_t *atom; + int old; + int mem; + + BUG_ON(t >= SCOUTFS_TRIGGER_NR); + atom = &triggers->atomics[t]; + + mem = atomic_read(atom); + if (likely(!mem)) + return 0; + + do { + old = mem; + mem = atomic_cmpxchg(atom, old, 0); + } while (mem && mem != old); + + return !!mem; +} + +int scoutfs_setup_triggers(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_triggers *triggers; + int ret; + int i; + + BUILD_BUG_ON(ARRAY_SIZE(names) != SCOUTFS_TRIGGER_NR); + + for (i = 0; i < ARRAY_SIZE(names); i++) { + if (WARN_ON(!names[i])) + return -EINVAL; + } + + triggers = kzalloc(sizeof(struct scoutfs_triggers), GFP_KERNEL); + if (!triggers) + return -ENOMEM; + + sbi->triggers = triggers; + + triggers->dir = debugfs_create_dir("trigger", sbi->debug_root); + if (!triggers->dir) { + ret = -ENOMEM; + goto out; + } + + for (i = 0; i < ARRAY_SIZE(triggers->atomics); i++) { + if (!debugfs_create_atomic_t(names[i], 0644, triggers->dir, + &triggers->atomics[i])) { + ret = -ENOMEM; + goto out; + } + } + + ret = 0; +out: + if (ret) + scoutfs_destroy_triggers(sb); + return ret; +} + +void scoutfs_destroy_triggers(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_triggers *triggers = sbi->triggers; + + if (triggers) { + if (triggers->dir) + debugfs_remove_recursive(triggers->dir); + kfree(triggers); + sbi->triggers = NULL; + } +} diff --git a/kmod/src/triggers.h b/kmod/src/triggers.h new file mode 100644 index 00000000..8796cd18 --- /dev/null +++ b/kmod/src/triggers.h @@ -0,0 +1,21 @@ +#ifndef _SCOUTFS_TRIGGERS_H_ +#define _SCOUTFS_TRIGGERS_H_ + +enum scoutfs_trigger { + SCOUTFS_TRIGGER_BTREE_STALE_READ, + SCOUTFS_TRIGGER_BTREE_ADVANCE_RING_HALF, + SCOUTFS_TRIGGER_HARD_STALE_ERROR, + SCOUTFS_TRIGGER_SEG_STALE_READ, + SCOUTFS_TRIGGER_STATFS_LOCK_PURGE, + SCOUTFS_TRIGGER_NR, +}; + +bool scoutfs_trigger_test_and_clear(struct super_block *sb, unsigned int t); + +#define scoutfs_trigger(sb, which) \ + scoutfs_trigger_test_and_clear(sb, SCOUTFS_TRIGGER_##which) + +int scoutfs_setup_triggers(struct super_block *sb); +void scoutfs_destroy_triggers(struct super_block *sb); + +#endif diff --git a/kmod/src/tseq.c b/kmod/src/tseq.c new file mode 100644 index 00000000..781d00f3 --- /dev/null +++ b/kmod/src/tseq.c @@ -0,0 +1,224 @@ +/* + * Copyright (C) 2018 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include +#include +#include + +#include "tseq.h" + +/* + * This trivial seq file wrapper takes care of the details of displaying + * a set of objects in seq file output. We use an augmented rbtree to + * add new objects at the next free file position. The caller takes + * care of the object life times, only debugfs file creation can fail. + */ + +static loff_t tseq_node_total(struct rb_node *node) +{ + struct scoutfs_tseq_entry *ent; + + if (node == NULL) + return 0; + + ent = rb_entry(node, struct scoutfs_tseq_entry, node); + return ent->total; +} + +static struct scoutfs_tseq_entry *tseq_rb_next(struct scoutfs_tseq_entry *ent) +{ + struct rb_node *node = rb_next(&ent->node); + + if (node == NULL) + return NULL; + + return rb_entry(node, struct scoutfs_tseq_entry, node); +} + +static loff_t tseq_compute_total(struct scoutfs_tseq_entry *ent) +{ + return 1 + tseq_node_total(ent->node.rb_left) + + tseq_node_total(ent->node.rb_right); +} + +RB_DECLARE_CALLBACKS(static, tseq_rb_callbacks, struct scoutfs_tseq_entry, + node, loff_t, total, tseq_compute_total) + +void scoutfs_tseq_tree_init(struct scoutfs_tseq_tree *tree, + scoutfs_tseq_show_t show) +{ + spin_lock_init(&tree->lock); + tree->root = RB_ROOT; + tree->show = show; +} + +/* + * Descend towards the leaf node that should be the parent for inserting + * a new entry. + * + * We use the augmented subtree totals to see when a left subtree has + * fewer entries than the current entry's pos which tells us that there + * is a lesser free pos. + * + * If there isn't a lesser free pos then we descend to the right and set + * the minimum possible pos to the pos after the entry we're traversing. + */ +void scoutfs_tseq_add(struct scoutfs_tseq_tree *tree, + struct scoutfs_tseq_entry *ins) +{ + struct scoutfs_tseq_entry *ent; + struct rb_node *parent; + struct rb_node **node; + loff_t min_pos; + + spin_lock(&tree->lock); + + node = &tree->root.rb_node; + parent = NULL; + min_pos = 0; + + while (*node) { + parent = *node; + ent = rb_entry(*node, struct scoutfs_tseq_entry, node); + + ent->total++; + + if (min_pos + tseq_node_total(ent->node.rb_left) < ent->pos) { + node = &ent->node.rb_left; + } else { + min_pos = ent->pos + 1; + node = &ent->node.rb_right; + } + } + + ins->pos = min_pos; + ins->total = 1; + rb_link_node(&ins->node, parent, node); + rb_insert_augmented(&ins->node, &tree->root, &tseq_rb_callbacks); + + spin_unlock(&tree->lock); +} + +static struct scoutfs_tseq_entry *tseq_pos_next(struct scoutfs_tseq_tree *tree, + loff_t pos) +{ + struct scoutfs_tseq_entry *next; + struct scoutfs_tseq_entry *ent; + struct rb_node *node; + + assert_spin_locked(&tree->lock); + + node = tree->root.rb_node; + next = NULL; + + while (node) { + ent = rb_entry(node, struct scoutfs_tseq_entry, node); + + if (pos < ent->pos) { + next = ent; + node = ent->node.rb_left; + } else if (pos > ent->pos) { + node = ent->node.rb_right; + } else { + return ent; + } + } + + return next; +} + +void scoutfs_tseq_del(struct scoutfs_tseq_tree *tree, + struct scoutfs_tseq_entry *ent) +{ + spin_lock(&tree->lock); + rb_erase_augmented(&ent->node, &tree->root, &tseq_rb_callbacks); + RB_CLEAR_NODE(&ent->node); + spin_unlock(&tree->lock); +} + +/* _stop is always called no matter what start returns */ +static void *scoutfs_tseq_seq_start(struct seq_file *m, loff_t *pos) + __acquires(tree->lock) +{ + struct scoutfs_tseq_tree *tree = m->private; + + spin_lock(&tree->lock); + + return tseq_pos_next(tree, *pos); +} + +static void *scoutfs_tseq_seq_next(struct seq_file *m, void *v, loff_t *pos) +{ + struct scoutfs_tseq_entry *ent = v; + + ent = tseq_rb_next(ent); + if (ent) + *pos = ent->pos; + return ent; +} + +static void scoutfs_tseq_seq_stop(struct seq_file *m, void *v) + __releases(tree->lock) +{ + struct scoutfs_tseq_tree *tree = m->private; + + spin_unlock(&tree->lock); +} + +static int scoutfs_tseq_seq_show(struct seq_file *m, void *v) +{ + struct scoutfs_tseq_tree *tree = m->private; + struct scoutfs_tseq_entry *ent = v; + + tree->show(m, ent); + return 0; +} + +static const struct seq_operations scoutfs_tseq_seq_ops = { + .start = scoutfs_tseq_seq_start, + .next = scoutfs_tseq_seq_next, + .stop = scoutfs_tseq_seq_stop, + .show = scoutfs_tseq_seq_show, +}; + +static int scoutfs_tseq_open(struct inode *inode, struct file *file) +{ + struct seq_file *m; + int ret; + + ret = seq_open(file, &scoutfs_tseq_seq_ops); + if (ret == 0) { + m = file->private_data; + m->private = inode->i_private; + } + return ret; +} + +static const struct file_operations scoutfs_tseq_fops = { + .open = scoutfs_tseq_open, + .release = seq_release, + .read = seq_read, + .llseek = seq_lseek, +}; + +/* + * This doesn't create any additional state so the returned dentry + * can be destroyed with the usual debugfs file calls. + */ +struct dentry *scoutfs_tseq_create(const char *name, struct dentry *parent, + struct scoutfs_tseq_tree *tree) +{ + return debugfs_create_file(name, S_IFREG|S_IRUSR, parent, tree, + &scoutfs_tseq_fops); +} diff --git a/kmod/src/tseq.h b/kmod/src/tseq.h new file mode 100644 index 00000000..d9b05a9e --- /dev/null +++ b/kmod/src/tseq.h @@ -0,0 +1,32 @@ +#ifndef _SCOUTFS_TSEQ_H_ +#define _SCOUTFS_TSEQ_H_ + +#include + +struct scoutfs_tseq_entry; +typedef void (*scoutfs_tseq_show_t)(struct seq_file *m, + struct scoutfs_tseq_entry *ent); + +struct scoutfs_tseq_tree { + spinlock_t lock; + struct rb_root root; + scoutfs_tseq_show_t show; +}; + +struct scoutfs_tseq_entry { + struct rb_node node; + loff_t pos; + loff_t total; +}; + +void scoutfs_tseq_tree_init(struct scoutfs_tseq_tree *tree, + scoutfs_tseq_show_t show); +void scoutfs_tseq_add(struct scoutfs_tseq_tree *tree, + struct scoutfs_tseq_entry *ent); +void scoutfs_tseq_del(struct scoutfs_tseq_tree *tree, + struct scoutfs_tseq_entry *ent); + +struct dentry *scoutfs_tseq_create(const char *name, struct dentry *parent, + struct scoutfs_tseq_tree *tree); + +#endif diff --git a/kmod/src/xattr.c b/kmod/src/xattr.c new file mode 100644 index 00000000..824b549b --- /dev/null +++ b/kmod/src/xattr.c @@ -0,0 +1,812 @@ +/* + * Copyright (C) 2018 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include +#include +#include + +#include "format.h" +#include "inode.h" +#include "key.h" +#include "super.h" +#include "item.h" +#include "forest.h" +#include "trans.h" +#include "xattr.h" +#include "lock.h" +#include "hash.h" +#include "scoutfs_trace.h" + +/* + * Extended attributes are packed into multiple smaller file system + * items. The common case only uses one item. + * + * The xattr keys contain the hash of the xattr name and a unique + * identifier used to differentiate xattrs whose names hash to the same + * value. xattr lookup has to walk all the xattrs with the matching + * name hash to compare the names. + * + * We use a rwsem in the inode to serialize modification of multiple + * items to make sure that we don't let readers race and see an + * inconsistent mix of the items that make up xattrs. + * + * XXX + * - add acl support and call generic xattr->handlers for SYSTEM + */ + +static u32 xattr_name_hash(const char *name, unsigned int name_len) +{ + return crc32c(U32_MAX, name, name_len); +} + +/* only compare names if the lens match, callers might not have both names */ +static u32 xattr_names_equal(const char *a_name, unsigned int a_len, + const char *b_name, unsigned int b_len) +{ + return a_len == b_len && memcmp(a_name, b_name, a_len) == 0; +} + +static unsigned int xattr_full_bytes(struct scoutfs_xattr *xat) +{ + return offsetof(struct scoutfs_xattr, + name[xat->name_len + le16_to_cpu(xat->val_len)]); +} + +static unsigned int xattr_nr_parts(struct scoutfs_xattr *xat) +{ + return SCOUTFS_XATTR_NR_PARTS(xat->name_len, + le16_to_cpu(xat->val_len)); +} + +static void init_xattr_key(struct scoutfs_key *key, u64 ino, u32 name_hash, + u64 id) +{ + *key = (struct scoutfs_key) { + .sk_zone = SCOUTFS_FS_ZONE, + .skx_ino = cpu_to_le64(ino), + .sk_type = SCOUTFS_XATTR_TYPE, + .skx_name_hash = cpu_to_le64(name_hash), + .skx_id = cpu_to_le64(id), + .skx_part = 0, + }; +} + +#define SCOUTFS_XATTR_PREFIX "scoutfs." +#define SCOUTFS_XATTR_PREFIX_LEN (sizeof(SCOUTFS_XATTR_PREFIX) - 1) + +static int unknown_prefix(const char *name) +{ + return strncmp(name, XATTR_USER_PREFIX, XATTR_USER_PREFIX_LEN) && + strncmp(name, XATTR_TRUSTED_PREFIX, XATTR_TRUSTED_PREFIX_LEN) && + strncmp(name, XATTR_SYSTEM_PREFIX, XATTR_SYSTEM_PREFIX_LEN) && + strncmp(name, XATTR_SECURITY_PREFIX, XATTR_SECURITY_PREFIX_LEN)&& + strncmp(name, SCOUTFS_XATTR_PREFIX, SCOUTFS_XATTR_PREFIX_LEN); +} + + +#define HIDE_TAG "hide." +#define SRCH_TAG "srch." +#define TAG_LEN (sizeof(HIDE_TAG) - 1) + +int scoutfs_xattr_parse_tags(const char *name, unsigned int name_len, + struct scoutfs_xattr_prefix_tags *tgs) +{ + bool found; + + memset(tgs, 0, sizeof(struct scoutfs_xattr_prefix_tags)); + + if ((name_len < (SCOUTFS_XATTR_PREFIX_LEN + TAG_LEN + 1)) || + strncmp(name, SCOUTFS_XATTR_PREFIX, SCOUTFS_XATTR_PREFIX_LEN)) + return 0; + name += SCOUTFS_XATTR_PREFIX_LEN; + + found = false; + for (;;) { + if (!strncmp(name, HIDE_TAG, TAG_LEN)) { + if (++tgs->hide == 0) + return -EINVAL; + } else if (!strncmp(name, SRCH_TAG, TAG_LEN)) { + if (++tgs->srch == 0) + return -EINVAL; + } else { + /* only reason to use scoutfs. is tags */ + if (!found) + return -EINVAL; + break; + } + name += TAG_LEN; + found = true; + } + + return 0; +} + +/* + * Find the next xattr and copy the key, xattr header, and as much of + * the name and value into the callers buffer as we can. Returns the + * number of bytes copied which include the header, name, and value and + * can be limited by the xattr length or the callers buffer. The caller + * is responsible for comparing their lengths, the header, and the + * returned length before safely using the xattr. + * + * If a name is provided then we'll iterate over items with a matching + * name_hash until we find a matching name. If we don't find a matching + * name then we return -ENOENT. + * + * If a name isn't provided then we'll return the next xattr from the + * given name_hash and id position. + * + * Returns -ENOENT if it didn't find a next item. + */ +static int get_next_xattr(struct inode *inode, struct scoutfs_key *key, + struct scoutfs_xattr *xat, unsigned int bytes, + const char *name, unsigned int name_len, + u64 name_hash, u64 id, struct scoutfs_lock *lock) +{ + struct super_block *sb = inode->i_sb; + struct scoutfs_key last; + u8 last_part; + int total; + u8 part; + int ret; + + /* need to be able to see the name we're looking for */ + if (WARN_ON_ONCE(name_len > 0 && bytes < offsetof(struct scoutfs_xattr, + name[name_len]))) + return -EINVAL; + + if (name_len) + name_hash = xattr_name_hash(name, name_len); + + init_xattr_key(key, scoutfs_ino(inode), name_hash, id); + init_xattr_key(&last, scoutfs_ino(inode), U32_MAX, U64_MAX); + + last_part = 0; + part = 0; + total = 0; + + for (;;) { + key->skx_part = part; + ret = scoutfs_item_next(sb, key, &last, + (void *)xat + total, bytes - total, + lock); + if (ret < 0) { + /* XXX corruption, ran out of parts */ + if (ret == -ENOENT && part > 0) + ret = -EIO; + break; + } + + trace_scoutfs_xattr_get_next_key(sb, key); + + /* XXX corruption */ + if (key->skx_part != part) { + ret = -EIO; + break; + } + + /* + * XXX corruption: We should have seen a valid header in + * the first part and if the next xattr name fits in our + * buffer then the item must have included it. + */ + if (part == 0 && + (ret < sizeof(struct scoutfs_xattr) || + (xat->name_len <= name_len && + ret < offsetof(struct scoutfs_xattr, + name[xat->name_len])) || + xat->name_len > SCOUTFS_XATTR_MAX_NAME_LEN || + le16_to_cpu(xat->val_len) > SCOUTFS_XATTR_MAX_VAL_LEN)) { + ret = -EIO; + break; + } + + if (part == 0 && name_len) { + /* ran out of names that could match */ + if (le64_to_cpu(key->skx_name_hash) != name_hash) { + ret = -ENOENT; + break; + } + + /* keep looking for our name */ + if (!xattr_names_equal(name, name_len, + xat->name, xat->name_len)) { + part = 0; + le64_add_cpu(&key->skx_id, 1); + continue; + } + + /* use the matching name we found */ + last_part = xattr_nr_parts(xat) - 1; + } + + total += ret; + if (total == bytes || part == last_part) { + /* copied as much as we could */ + ret = total; + break; + } + part++; + } + + return ret; +} + +/* + * Create all the items associated with the given xattr. If this + * returns an error it will have already cleaned up any items it created + * before seeing the error. + */ +static int create_xattr_items(struct inode *inode, u64 id, + struct scoutfs_xattr *xat, unsigned int bytes, + struct scoutfs_lock *lock) +{ + struct super_block *sb = inode->i_sb; + struct scoutfs_key key; + unsigned int part_bytes; + unsigned int total; + int ret; + + init_xattr_key(&key, scoutfs_ino(inode), + xattr_name_hash(xat->name, xat->name_len), id); + + total = 0; + ret = 0; + while (total < bytes) { + part_bytes = min_t(unsigned int, bytes - total, + SCOUTFS_XATTR_MAX_PART_SIZE); + + ret = scoutfs_item_create(sb, &key, + (void *)xat + total, part_bytes, + lock); + if (ret) { + while (key.skx_part-- > 0) + scoutfs_item_delete(sb, &key, lock); + break; + } + + total += part_bytes; + key.skx_part++; + } + + return ret; +} + +/* + * Delete the items that make up the given xattr. If this returns an + * error then no items have been deleted. + */ +static int delete_xattr_items(struct inode *inode, u32 name_hash, u64 id, + u8 nr_parts, struct scoutfs_lock *lock) +{ + struct super_block *sb = inode->i_sb; + struct scoutfs_key key; + int ret = 0; + int i; + + init_xattr_key(&key, scoutfs_ino(inode), name_hash, id); + + /* dirty additional existing old items */ + for (i = 1; i < nr_parts; i++) { + key.skx_part = i; + ret = scoutfs_item_dirty(sb, &key, lock); + if (ret) + goto out; + } + + for (i = 0; i < nr_parts; i++) { + key.skx_part = i; + ret = scoutfs_item_delete(sb, &key, lock); + if (ret) + break; + } +out: + return ret; +} + +/* + * The caller needs to overwrite existing old xattr items with new + * items. We carefully stage the changes so that we can always unwind + * to the original items if we return an error. Both items have at + * least one part. Either the old or new can have more parts. We dirty + * and create first because we can always unwind those. We delete last + * after dirtying so that it can't fail and we don't have to restore the + * deleted items. + */ +static int change_xattr_items(struct inode *inode, u64 id, + struct scoutfs_xattr *new_xat, + unsigned int new_bytes, u8 new_parts, + u8 old_parts, struct scoutfs_lock *lock) +{ + struct super_block *sb = inode->i_sb; + struct scoutfs_key key; + int last_created = -1; + int bytes; + int off; + int i; + int ret; + + init_xattr_key(&key, scoutfs_ino(inode), + xattr_name_hash(new_xat->name, new_xat->name_len), id); + + /* dirty existing old items */ + for (i = 0; i < old_parts; i++) { + key.skx_part = i; + ret = scoutfs_item_dirty(sb, &key, lock); + if (ret) + goto out; + } + + /* create any new items past the old */ + for (i = old_parts; i < new_parts; i++) { + off = i * SCOUTFS_XATTR_MAX_PART_SIZE; + bytes = min_t(unsigned int, new_bytes - off, + SCOUTFS_XATTR_MAX_PART_SIZE); + + key.skx_part = i; + ret = scoutfs_item_create(sb, &key, (void *)new_xat + off, + bytes, lock); + if (ret) + goto out; + + last_created = i; + } + + /* update dirtied overlapping existing items, last partial first */ + for (i = old_parts - 1; i >= 0; i--) { + off = i * SCOUTFS_XATTR_MAX_PART_SIZE; + bytes = min_t(unsigned int, new_bytes - off, + SCOUTFS_XATTR_MAX_PART_SIZE); + + key.skx_part = i; + ret = scoutfs_item_update(sb, &key, (void *)new_xat + off, + bytes, lock); + /* only last partial can fail, then we unwind created */ + if (ret < 0) + goto out; + } + + /* delete any dirtied old items past new */ + for (i = new_parts; i < old_parts; i++) { + key.skx_part = i; + scoutfs_item_delete(sb, &key, lock); + } + + ret = 0; +out: + if (ret < 0) { + /* delete any newly created items */ + for (i = old_parts; i <= last_created; i++) { + key.skx_part = i; + scoutfs_item_delete(sb, &key, lock); + } + } + return ret; +} + +/* + * Copy the value for the given xattr name into the caller's buffer, if it + * fits. Return the bytes copied or -ERANGE if it doesn't fit. + */ +ssize_t scoutfs_getxattr(struct dentry *dentry, const char *name, void *buffer, + size_t size) +{ + struct inode *inode = dentry->d_inode; + struct scoutfs_inode_info *si = SCOUTFS_I(inode); + struct super_block *sb = inode->i_sb; + struct scoutfs_xattr *xat = NULL; + struct scoutfs_lock *lck = NULL; + struct scoutfs_key key; + unsigned int bytes; + size_t name_len; + int ret; + + if (unknown_prefix(name)) + return -EOPNOTSUPP; + + name_len = strlen(name); + if (name_len > SCOUTFS_XATTR_MAX_NAME_LEN) + return -ENODATA; + + /* only need enough for caller's name and value sizes */ + bytes = sizeof(struct scoutfs_xattr) + name_len + size; + xat = __vmalloc(bytes, GFP_NOFS, PAGE_KERNEL); + if (!xat) + return -ENOMEM; + + ret = scoutfs_lock_inode(sb, SCOUTFS_LOCK_READ, 0, inode, &lck); + if (ret) + goto out; + + down_read(&si->xattr_rwsem); + + ret = get_next_xattr(inode, &key, xat, bytes, + name, name_len, 0, 0, lck); + + up_read(&si->xattr_rwsem); + scoutfs_unlock(sb, lck, SCOUTFS_LOCK_READ); + + if (ret < 0) { + if (ret == -ENOENT) + ret = -ENODATA; + goto out; + } + + /* the caller just wants to know the size */ + if (size == 0) { + ret = le16_to_cpu(xat->val_len); + goto out; + } + + /* the caller's buffer wasn't big enough */ + if (size < le16_to_cpu(xat->val_len)) { + ret = -ERANGE; + goto out; + } + + /* XXX corruption, the items didn't match the header */ + if (ret < xattr_full_bytes(xat)) { + ret = -EIO; + goto out; + } + + ret = le16_to_cpu(xat->val_len); + memcpy(buffer, &xat->name[xat->name_len], ret); +out: + vfree(xat); + return ret; +} + +/* + * The confusing swiss army knife of creating, modifying, and deleting + * xattrs. + * + * This always removes the old existing xattr items. + * + * If the value pointer is set then we're adding a new xattr. The flags + * cause creation to fail if the xattr already exists (_CREATE) or + * doesn't already exist (_REPLACE). xattrs can have a zero length + * value. + */ +static int scoutfs_xattr_set(struct dentry *dentry, const char *name, + const void *value, size_t size, int flags) +{ + struct inode *inode = dentry->d_inode; + struct scoutfs_inode_info *si = SCOUTFS_I(inode); + struct super_block *sb = inode->i_sb; + const u64 ino = scoutfs_ino(inode); + struct scoutfs_xattr_prefix_tags tgs; + struct scoutfs_xattr *xat = NULL; + struct scoutfs_lock *lck = NULL; + size_t name_len = strlen(name); + struct scoutfs_key key; + bool undo_srch = false; + LIST_HEAD(ind_locks); + u8 found_parts; + unsigned int bytes; + u64 ind_seq; + u64 hash = 0; + u64 id = 0; + int ret; + int err; + + trace_scoutfs_xattr_set(sb, name_len, value, size, flags); + + /* mirror the syscall's errors for large names and values */ + if (name_len > SCOUTFS_XATTR_MAX_NAME_LEN) + return -ERANGE; + if (value && size > SCOUTFS_XATTR_MAX_VAL_LEN) + return -E2BIG; + + if (((flags & XATTR_CREATE) && (flags & XATTR_REPLACE)) || + (flags & ~(XATTR_CREATE | XATTR_REPLACE))) + return -EINVAL; + + if (unknown_prefix(name)) + return -EOPNOTSUPP; + + if (scoutfs_xattr_parse_tags(name, name_len, &tgs) != 0) + return -EINVAL; + + if ((tgs.hide || tgs.srch) && !capable(CAP_SYS_ADMIN)) + return -EPERM; + + bytes = sizeof(struct scoutfs_xattr) + name_len + size; + xat = __vmalloc(bytes, GFP_NOFS, PAGE_KERNEL); + if (!xat) { + ret = -ENOMEM; + goto out; + } + + ret = scoutfs_lock_inode(sb, SCOUTFS_LOCK_WRITE, + SCOUTFS_LKF_REFRESH_INODE, inode, &lck); + if (ret) + goto out; + + down_write(&si->xattr_rwsem); + + /* find an existing xattr to delete */ + ret = get_next_xattr(inode, &key, xat, + sizeof(struct scoutfs_xattr) + name_len, + name, name_len, 0, 0, lck); + if (ret < 0 && ret != -ENOENT) + goto unlock; + + /* check existence constraint flags */ + if (ret == -ENOENT && (flags & XATTR_REPLACE)) { + ret = -ENODATA; + goto unlock; + } else if (ret >= 0 && (flags & XATTR_CREATE)) { + ret = -EEXIST; + goto unlock; + } + + /* not an error to delete something that doesn't exist */ + if (ret == -ENOENT && !value) { + ret = 0; + goto unlock; + } + + /* found fields in key will also be used */ + found_parts = ret >= 0 ? xattr_nr_parts(xat) : 0; + + /* prepare our xattr */ + if (value) { + if (found_parts) + id = le64_to_cpu(key.skx_id); + else + id = si->next_xattr_id++; + xat->name_len = name_len; + xat->val_len = cpu_to_le16(size); + memset(xat->__pad, 0, sizeof(xat->__pad)); + memcpy(xat->name, name, name_len); + memcpy(&xat->name[xat->name_len], value, size); + } + +retry: + ret = scoutfs_inode_index_start(sb, &ind_seq) ?: + scoutfs_inode_index_prepare(sb, &ind_locks, inode, false) ?: + scoutfs_inode_index_try_lock_hold(sb, &ind_locks, ind_seq, + SIC_XATTR_SET(found_parts, + value != NULL, + name_len, size)); + if (ret > 0) + goto retry; + if (ret) + goto unlock; + + ret = scoutfs_dirty_inode_item(inode, lck); + if (ret < 0) + goto release; + + if (tgs.srch && !(found_parts && value)) { + if (found_parts) + id = le64_to_cpu(key.skx_id); + hash = scoutfs_hash64(name, name_len); + ret = scoutfs_forest_srch_add(sb, hash, ino, id); + if (ret < 0) + goto release; + undo_srch = true; + } + + if (found_parts && value) + ret = change_xattr_items(inode, id, xat, bytes, + xattr_nr_parts(xat), found_parts, lck); + else if (found_parts) + ret = delete_xattr_items(inode, le64_to_cpu(key.skx_name_hash), + le64_to_cpu(key.skx_id), found_parts, + lck); + else + ret = create_xattr_items(inode, id, xat, bytes, lck); + if (ret < 0) + goto release; + + /* XXX do these want i_mutex or anything? */ + inode_inc_iversion(inode); + inode->i_ctime = CURRENT_TIME; + scoutfs_update_inode_item(inode, lck, &ind_locks); + ret = 0; + +release: + if (ret < 0 && undo_srch) { + err = scoutfs_forest_srch_add(sb, hash, ino, id); + BUG_ON(err); + } + + scoutfs_release_trans(sb); + scoutfs_inode_index_unlock(sb, &ind_locks); +unlock: + up_write(&si->xattr_rwsem); + scoutfs_unlock(sb, lck, SCOUTFS_LOCK_WRITE); +out: + vfree(xat); + + return ret; +} + +int scoutfs_setxattr(struct dentry *dentry, const char *name, + const void *value, size_t size, int flags) +{ + if (size == 0) + value = ""; /* set empty value */ + + return scoutfs_xattr_set(dentry, name, value, size, flags); +} + +int scoutfs_removexattr(struct dentry *dentry, const char *name) +{ + return scoutfs_xattr_set(dentry, name, NULL, 0, XATTR_REPLACE); +} + +ssize_t scoutfs_list_xattrs(struct inode *inode, char *buffer, + size_t size, __u32 *hash_pos, __u64 *id_pos, + bool e_range, bool show_hidden) +{ + struct scoutfs_inode_info *si = SCOUTFS_I(inode); + struct super_block *sb = inode->i_sb; + struct scoutfs_xattr_prefix_tags tgs; + struct scoutfs_xattr *xat = NULL; + struct scoutfs_lock *lck = NULL; + struct scoutfs_key key; + unsigned int bytes; + ssize_t total = 0; + u32 name_hash = 0; + bool is_hidden; + u64 id = 0; + int ret; + + if (hash_pos) + name_hash = *hash_pos; + if (id_pos) + id = *id_pos; + + /* need a buffer large enough for all possible names */ + bytes = sizeof(struct scoutfs_xattr) + SCOUTFS_XATTR_MAX_NAME_LEN; + xat = kmalloc(bytes, GFP_NOFS); + if (!xat) { + ret = -ENOMEM; + goto out; + } + + ret = scoutfs_lock_inode(sb, SCOUTFS_LOCK_READ, 0, inode, &lck); + if (ret) + goto out; + + down_read(&si->xattr_rwsem); + + for (;;) { + ret = get_next_xattr(inode, &key, xat, bytes, + NULL, 0, name_hash, id, lck); + if (ret < 0) { + if (ret == -ENOENT) + ret = total; + break; + } + + is_hidden = scoutfs_xattr_parse_tags(xat->name, xat->name_len, + &tgs) == 0 && tgs.hide; + + if (show_hidden == is_hidden) { + if (size) { + if ((total + xat->name_len + 1) > size) { + if (e_range) + ret = -ERANGE; + else + ret = total; + break; + } + + memcpy(buffer, xat->name, xat->name_len); + buffer += xat->name_len; + *(buffer++) = '\0'; + } + + total += xat->name_len + 1; + } + + name_hash = le64_to_cpu(key.skx_name_hash); + id = le64_to_cpu(key.skx_id) + 1; + } + + up_read(&si->xattr_rwsem); + scoutfs_unlock(sb, lck, SCOUTFS_LOCK_READ); +out: + kfree(xat); + + if (hash_pos) + *hash_pos = name_hash; + if (id_pos) + *id_pos = id; + + return ret; +} + +ssize_t scoutfs_listxattr(struct dentry *dentry, char *buffer, size_t size) +{ + struct inode *inode = dentry->d_inode; + + return scoutfs_list_xattrs(inode, buffer, size, + NULL, NULL, true, false); +} + +/* + * Delete all the xattr items associated with this inode. The inode is + * dead so we don't need the xattr rwsem. + */ +int scoutfs_xattr_drop(struct super_block *sb, u64 ino, + struct scoutfs_lock *lock) +{ + struct scoutfs_xattr_prefix_tags tgs; + struct scoutfs_xattr *xat = NULL; + struct scoutfs_key last; + struct scoutfs_key key; + bool release = false; + unsigned int bytes; + u64 hash; + int ret; + + /* need a buffer large enough for all possible names */ + bytes = sizeof(struct scoutfs_xattr) + SCOUTFS_XATTR_MAX_NAME_LEN; + xat = kmalloc(bytes, GFP_NOFS); + if (!xat) { + ret = -ENOMEM; + goto out; + } + + init_xattr_key(&key, ino, 0, 0); + init_xattr_key(&last, ino, U32_MAX, U64_MAX); + + for (;;) { + ret = scoutfs_item_next(sb, &key, &last, (void *)xat, bytes, + lock); + if (ret < 0) { + if (ret == -ENOENT) + ret = 0; + break; + } + + if (key.skx_part != 0 || + scoutfs_xattr_parse_tags(xat->name, xat->name_len, + &tgs) != 0) + memset(&tgs, 0, sizeof(tgs)); + + ret = scoutfs_hold_trans(sb, SIC_EXACT(2, 0)); + if (ret < 0) + break; + release = true; + + ret = scoutfs_item_delete(sb, &key, lock); + if (ret < 0) + break; + + if (tgs.srch) { + hash = scoutfs_hash64(xat->name, xat->name_len); + ret = scoutfs_forest_srch_add(sb, hash, ino, + le64_to_cpu(key.skx_id)); + if (ret < 0) + break; + } + + scoutfs_release_trans(sb); + release = false; + + /* don't need to inc, next won't see deleted item */ + } + + if (release) + scoutfs_release_trans(sb); + kfree(xat); +out: + return ret; +} diff --git a/kmod/src/xattr.h b/kmod/src/xattr.h new file mode 100644 index 00000000..39313801 --- /dev/null +++ b/kmod/src/xattr.h @@ -0,0 +1,25 @@ +#ifndef _SCOUTFS_XATTR_H_ +#define _SCOUTFS_XATTR_H_ + +ssize_t scoutfs_getxattr(struct dentry *dentry, const char *name, void *buffer, + size_t size); +int scoutfs_setxattr(struct dentry *dentry, const char *name, + const void *value, size_t size, int flags); +int scoutfs_removexattr(struct dentry *dentry, const char *name); +ssize_t scoutfs_listxattr(struct dentry *dentry, char *buffer, size_t size); +ssize_t scoutfs_list_xattrs(struct inode *inode, char *buffer, + size_t size, __u32 *hash_pos, __u64 *id_pos, + bool e_range, bool show_hidden); + +int scoutfs_xattr_drop(struct super_block *sb, u64 ino, + struct scoutfs_lock *lock); + +struct scoutfs_xattr_prefix_tags { + unsigned long hide:1, + srch:1; +}; + +int scoutfs_xattr_parse_tags(const char *name, unsigned int name_len, + struct scoutfs_xattr_prefix_tags *tgs); + +#endif